http://git-wip-us.apache.org/repos/asf/lens/blob/87049563/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java index d938ff7..e415a5a 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java @@ -19,12 +19,12 @@ package org.apache.lens.cube.metadata; -import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY; -import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY; -import static org.apache.lens.cube.metadata.UpdatePeriod.MONTHLY; +import static org.apache.lens.cube.metadata.MetastoreUtil.*; +import static org.apache.lens.cube.metadata.UpdatePeriod.*; +import static org.apache.lens.cube.parse.CubeTestSetup.DateOffsetProvider; +import static org.apache.lens.server.api.util.LensUtil.getHashMap; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.fail; +import static org.testng.Assert.*; import java.util.*; @@ -37,6 +37,7 @@ import org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline; import org.apache.lens.cube.metadata.timeline.TestPartitionTimelines; import org.apache.lens.cube.parse.TimeRange; import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.util.LensUtil; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; @@ -50,13 +51,14 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.TextInputFormat; -import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; public class TestCubeMetastoreClient { @@ -71,35 +73,35 @@ public class TestCubeMetastoreClient { private static Set<String> dimensions; private static Set<CubeMeasure> cubeMeasures; private static Set<CubeDimAttribute> cubeDimensions; + private static Set<UpdatePeriod> hourlyAndDaily = Sets.newHashSet(HOURLY, DAILY); private static final String CUBE_NAME = "testMetastoreCube"; private static final String CUBE_NAME_WITH_PROPS = "testMetastoreCubeWithProps"; private static final String DERIVED_CUBE_NAME = "derivedTestMetastoreCube"; private static final String DERIVED_CUBE_NAME_WITH_PROPS = "derivedTestMetastoreCubeWithProps"; - private static final Map<String, String> CUBE_PROPERTIES = new HashMap<String, String>(); - private static Date now; - private static Date nowPlus1; - private static Date nowPlus2; - private static Date nowPlus3; - private static Date nowMinus1; - private static Date nowMinus2; - private static Date nowMinus3; - private static Date nowMinus4; - private static Date nowMinus5; + private static final Map<String, String> CUBE_PROPERTIES = new HashMap<>(); private static HiveConf conf = new HiveConf(TestCubeMetastoreClient.class); private static FieldSchema dtPart = new FieldSchema(getDatePartitionKey(), serdeConstants.STRING_TYPE_NAME, "date partition"); + private static ArrayList<String> datePartKeySingleton = Lists.newArrayList(getDatePartitionKey()); + private static ArrayList<FieldSchema> datePartSingleton = Lists.newArrayList(getDatePartition()); + private static Map<String, String> emptyHashMap = ImmutableMap.copyOf(LensUtil.<String, String>getHashMap()); private static String c1 = "C1"; private static String c2 = "C2"; private static String c3 = "C3"; private static String c4 = "C4"; private static Dimension zipDim, cityDim, stateDim, countryDim; - private static Set<CubeDimAttribute> zipAttrs = new HashSet<CubeDimAttribute>(); - private static Set<CubeDimAttribute> cityAttrs = new HashSet<CubeDimAttribute>(); - private static Set<CubeDimAttribute> stateAttrs = new HashSet<CubeDimAttribute>(); - private static Set<CubeDimAttribute> countryAttrs = new HashSet<CubeDimAttribute>(); - private static Set<ExprColumn> cubeExpressions = new HashSet<ExprColumn>(); - private static Set<JoinChain> joinChains = new HashSet<JoinChain>(); - private static Set<ExprColumn> dimExpressions = new HashSet<ExprColumn>(); + private static Set<CubeDimAttribute> zipAttrs = new HashSet<>(); + private static Set<CubeDimAttribute> cityAttrs = new HashSet<>(); + private static Set<CubeDimAttribute> stateAttrs = new HashSet<>(); + private static Set<CubeDimAttribute> countryAttrs = new HashSet<>(); + private static Set<ExprColumn> cubeExpressions = new HashSet<>(); + private static Set<JoinChain> joinChains = new HashSet<>(); + private static Set<ExprColumn> dimExpressions = new HashSet<>(); + private static DateOffsetProvider dateOffsetProvider = new DateOffsetProvider(HOURLY); + + public static Date getDateWithOffset(int i) { + return dateOffsetProvider.get(i); + } /** * Get the date partition as field schema @@ -107,7 +109,7 @@ public class TestCubeMetastoreClient { * @return FieldSchema */ public static FieldSchema getDatePartition() { - return TestCubeMetastoreClient.dtPart; + return dtPart; } /** @@ -119,30 +121,18 @@ public class TestCubeMetastoreClient { return StorageConstants.DATE_PARTITION_KEY; } + private static HashMap<String, Date> getTimePartitionByOffsets(Object... args) { + for (int i = 1; i < args.length; i += 2) { + if (args[i] instanceof Integer) { + args[i] = getDateWithOffset((Integer) args[i]); + } + } + return getHashMap(args); + } + @BeforeClass public static void setup() throws HiveException, AlreadyExistsException, LensException { SessionState.start(conf); - now = new Date(); - Calendar cal = Calendar.getInstance(); - cal.setTime(now); - cal.add(Calendar.HOUR_OF_DAY, 1); - nowPlus1 = cal.getTime(); - cal.add(Calendar.HOUR_OF_DAY, 1); - nowPlus2 = cal.getTime(); - cal.add(Calendar.HOUR_OF_DAY, 1); - nowPlus3 = cal.getTime(); - cal.setTime(now); - cal.add(Calendar.HOUR, -1); - nowMinus1 = cal.getTime(); - cal.add(Calendar.HOUR, -1); - nowMinus2 = cal.getTime(); - cal.add(Calendar.HOUR, -1); - nowMinus3 = cal.getTime(); - cal.add(Calendar.HOUR, -1); - nowMinus4 = cal.getTime(); - cal.add(Calendar.HOUR, -1); - nowMinus5 = cal.getTime(); - Database database = new Database(); database.setName(TestCubeMetastoreClient.class.getSimpleName()); Hive.get(conf).createDatabase(database); @@ -157,7 +147,7 @@ public class TestCubeMetastoreClient { // Drop the cube client.dropCube(CUBE_NAME); client = CubeMetastoreClient.getInstance(conf); - Assert.assertFalse(client.tableExists(CUBE_NAME)); + assertFalse(client.tableExists(CUBE_NAME)); Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true); CubeMetastoreClient.close(); @@ -165,7 +155,7 @@ public class TestCubeMetastoreClient { private static void defineCube(String cubeName, String cubeNameWithProps, String derivedCubeName, String derivedCubeNameWithProps) throws LensException { - cubeMeasures = new HashSet<CubeMeasure>(); + cubeMeasures = new HashSet<>(); cubeMeasures.add(new ColumnMeasure( new FieldSchema("msr1", "int", "first measure"), null, null, null, null, null, null, null, 0.0, 9999.0)); cubeMeasures.add(new ColumnMeasure( @@ -176,19 +166,19 @@ public class TestCubeMetastoreClient { new FieldSchema("msr4", "bigint", "fourth measure"), "Measure4", null, "COUNT", null)); cubeMeasures.add(new ColumnMeasure( new FieldSchema("msrstarttime", "int", "measure with start time"), - "Measure With Starttime", null, null, null, now, null, null, 0.0, 999999.0)); + "Measure With Starttime", null, null, null, getDateWithOffset(0), null, null, 0.0, 999999.0)); cubeMeasures.add(new ColumnMeasure( new FieldSchema("msrendtime", "float", "measure with end time"), - "Measure With Endtime", null, "SUM", "RS", now, now, null)); + "Measure With Endtime", null, "SUM", "RS", getDateWithOffset(0), getDateWithOffset(0), null)); cubeMeasures.add(new ColumnMeasure( new FieldSchema("msrcost", "double", "measure with cost"), "Measure With cost", - null, "MAX", null, now, now, 100.0)); + null, "MAX", null, getDateWithOffset(0), getDateWithOffset(0), 100.0)); cubeMeasures.add(new ColumnMeasure( new FieldSchema("msrcost2", "bigint", "measure with cost"), "Measure With cost2", null, "MAX", null, null, null, 100.0, 0.0, 999999999999999999999999999.0)); - cubeDimensions = new HashSet<CubeDimAttribute>(); - List<CubeDimAttribute> locationHierarchy = new ArrayList<CubeDimAttribute>(); + cubeDimensions = new HashSet<>(); + List<CubeDimAttribute> locationHierarchy = new ArrayList<>(); locationHierarchy.add(new ReferencedDimAtrribute(new FieldSchema("zipcode", "int", "zip"), "Zip refer", new TableReference("zipdim", "zipcode"))); locationHierarchy.add(new ReferencedDimAtrribute(new FieldSchema("cityid", "int", "city"), "City refer", @@ -230,13 +220,14 @@ public class TestCubeMetastoreClient { cubeExpressions.add(new ExprColumn(new FieldSchema("substrexpr", "string", "a subt string expression"), "SUBSTR EXPR", expr1, expr2)); - List<CubeDimAttribute> locationHierarchyWithStartTime = new ArrayList<CubeDimAttribute>(); + List<CubeDimAttribute> locationHierarchyWithStartTime = new ArrayList<>(); locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new FieldSchema("zipcode2", "int", "zip"), - "Zip refer2", new TableReference("zipdim", "zipcode"), now, now, 100.0, true, 1000L)); + "Zip refer2", new TableReference("zipdim", "zipcode"), getDateWithOffset(0), getDateWithOffset(0), + 100.0, true, 1000L)); locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new FieldSchema("cityid2", "int", "city"), - "City refer2", new TableReference("citydim", "id"), now, null, null)); + "City refer2", new TableReference("citydim", "id"), getDateWithOffset(0), null, null)); locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new FieldSchema("stateid2", "int", "state"), - "state refer2", new TableReference("statedim", "id"), now, null, 100.0)); + "state refer2", new TableReference("statedim", "id"), getDateWithOffset(0), null, 100.0)); locationHierarchyWithStartTime.add(new ReferencedDimAtrribute(new FieldSchema("countryid2", "int", "country"), "Country refer2", new TableReference("countrydim", "id"), null, null, null)); locationHierarchyWithStartTime.add(new BaseDimAttribute(new FieldSchema("regionname2", "string", "region"), @@ -245,11 +236,11 @@ public class TestCubeMetastoreClient { cubeDimensions .add(new HierarchicalDimAttribute("location2", "localtion hierarchy2", locationHierarchyWithStartTime)); cubeDimensions.add(new BaseDimAttribute(new FieldSchema("dim1startTime", "string", "basedim"), - "Dim With starttime", now, null, 100.0)); + "Dim With starttime", getDateWithOffset(0), null, 100.0)); cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2start", "string", "ref dim"), - "Dim2 with starttime", new TableReference("testdim2", "id"), now, now, 100.0)); + "Dim2 with starttime", new TableReference("testdim2", "id"), getDateWithOffset(0), getDateWithOffset(0), 100.0)); - List<TableReference> multiRefs = new ArrayList<TableReference>(); + List<TableReference> multiRefs = new ArrayList<>(); multiRefs.add(new TableReference("testdim2", "id")); multiRefs.add(new TableReference("testdim3", "id")); multiRefs.add(new TableReference("testdim4", "id")); @@ -257,20 +248,20 @@ public class TestCubeMetastoreClient { cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim3", "string", "multi ref dim"), "Dim3 refer", multiRefs)); cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim3start", "string", "multi ref dim"), - "Dim3 with starttime", multiRefs, now, null, 100.0)); + "Dim3 with starttime", multiRefs, getDateWithOffset(0), null, 100.0)); cubeDimensions.add(new BaseDimAttribute(new FieldSchema("region", "string", "region dim"), "region", null, null, null, null, regions)); cubeDimensions.add(new BaseDimAttribute(new FieldSchema("regionstart", "string", "region dim"), - "Region with starttime", now, null, 100.0, null, regions)); + "Region with starttime", getDateWithOffset(0), null, 100.0, null, regions)); JoinChain zipCity = new JoinChain("cityFromZip", "Zip City", "zip city desc"); - List<TableReference> chain = new ArrayList<TableReference>(); + List<TableReference> chain = new ArrayList<>(); chain.add(new TableReference(cubeName, "zipcode")); chain.add(new TableReference("zipdim", "zipcode")); chain.add(new TableReference("zipdim", "cityid")); chain.add(new TableReference("citydim", "id")); zipCity.addPath(chain); - List<TableReference> chain2 = new ArrayList<TableReference>(); + List<TableReference> chain2 = new ArrayList<>(); chain2.add(new TableReference(cubeName, "zipcode2")); chain2.add(new TableReference("zipdim", "zipcode")); chain2.add(new TableReference("zipdim", "cityid")); @@ -278,23 +269,16 @@ public class TestCubeMetastoreClient { zipCity.addPath(chain2); joinChains.add(zipCity); JoinChain cityChain = new JoinChain("city", "Cube City", "cube city desc"); - chain = new ArrayList<TableReference>(); + chain = new ArrayList<>(); chain.add(new TableReference(cubeName, "cityid")); chain.add(new TableReference("citydim", "id")); cityChain.addPath(chain); joinChains.add(cityChain); cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("zipcityname", "string", "zip city name"), "Zip city name", "cityFromZip", "name", null, null, null)); - cube = new Cube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, - new HashMap<String, String>(), 0.0); - measures = new HashSet<String>(); - measures.add("msr1"); - measures.add("msr2"); - measures.add("msr3"); - dimensions = new HashSet<String>(); - dimensions.add("dim1"); - dimensions.add("dim2"); - dimensions.add("dim3"); + cube = new Cube(cubeName, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, emptyHashMap, 0.0); + measures = Sets.newHashSet("msr1", "msr2", "msr3"); + dimensions = Sets.newHashSet("dim1", "dim2", "dim3"); derivedCube = new DerivedCube(derivedCubeName, measures, dimensions, cube); CUBE_PROPERTIES.put(MetastoreUtil.getCubeTimedDimensionListKey(cubeNameWithProps), "dt,mydate"); @@ -310,9 +294,8 @@ public class TestCubeMetastoreClient { zipAttrs.add(new BaseDimAttribute(new FieldSchema("zipcode", "int", "code"))); zipAttrs.add(new BaseDimAttribute(new FieldSchema("f1", "string", "field1"))); zipAttrs.add(new BaseDimAttribute(new FieldSchema("f2", "string", "field1"))); - List<TableReference> stateRefs = new ArrayList<TableReference>(); - stateRefs.add(new TableReference("statedim", "id")); - stateRefs.add(new TableReference("stateWeatherDim", "id")); + List<TableReference> stateRefs = Lists.newArrayList(new TableReference("statedim", "id"), + new TableReference("stateWeatherDim", "id")); zipAttrs.add(new ReferencedDimAtrribute(new FieldSchema("stateid", "int", "state id"), "State refer", stateRefs)); zipAttrs.add(new ReferencedDimAtrribute(new FieldSchema("cityid", "int", "city id"), "City refer", new TableReference("citydim", "id"))); @@ -330,8 +313,7 @@ public class TestCubeMetastoreClient { new ExprSpec("state_and_country", null, null))); dimExpressions.add(new ExprColumn(new FieldSchema("CityAddress", "string", "city with state and city and zip"), "City Address", "concat(citydim.name, \":\", statedim.name, \":\", countrydim.name, \":\", zipcode.code)")); - Map<String, String> dimProps = new HashMap<String, String>(); - dimProps.put(MetastoreUtil.getDimTimedDimensionKey("citydim"), TestCubeMetastoreClient.getDatePartitionKey()); + Map<String, String> dimProps = getHashMap(getDimTimedDimensionKey("citydim"), getDatePartitionKey()); cityDim = new Dimension("citydim", cityAttrs, dimExpressions, dimProps, 0L); // Define state table @@ -354,19 +336,19 @@ public class TestCubeMetastoreClient { public void testStorage() throws Exception { Storage hdfsStorage = new HDFSStorage(c1); client.createStorage(hdfsStorage); - assertEquals(1, client.getAllStorages().size()); + assertEquals(client.getAllStorages().size(), 1); Storage hdfsStorage2 = new HDFSStorage(c2); client.createStorage(hdfsStorage2); - assertEquals(2, client.getAllStorages().size()); + assertEquals(client.getAllStorages().size(), 2); Storage hdfsStorage3 = new HDFSStorage(c3); client.createStorage(hdfsStorage3); - assertEquals(3, client.getAllStorages().size()); + assertEquals(client.getAllStorages().size(), 3); Storage hdfsStorage4 = new HDFSStorage(c4); client.createStorage(hdfsStorage4); - assertEquals(4, client.getAllStorages().size()); + assertEquals(client.getAllStorages().size(), 4); assertEquals(hdfsStorage, client.getStorage(c1)); assertEquals(hdfsStorage2, client.getStorage(c2)); @@ -382,9 +364,9 @@ public class TestCubeMetastoreClient { client.createDimension(countryDim); assertEquals(client.getAllDimensions().size(), 4); - Assert.assertTrue(client.tableExists(cityDim.getName())); - Assert.assertTrue(client.tableExists(stateDim.getName())); - Assert.assertTrue(client.tableExists(countryDim.getName())); + assertTrue(client.tableExists(cityDim.getName())); + assertTrue(client.tableExists(stateDim.getName())); + assertTrue(client.tableExists(countryDim.getName())); validateDim(zipDim, zipAttrs, "zipcode", "stateid"); validateDim(cityDim, cityAttrs, "id", "stateid"); @@ -395,8 +377,8 @@ public class TestCubeMetastoreClient { Dimension city = client.getDimension(cityDim.getName()); assertEquals(dimExpressions.size(), city.getExpressions().size()); assertEquals(dimExpressions.size(), city.getExpressionNames().size()); - Assert.assertNotNull(city.getExpressionByName("stateAndCountry")); - Assert.assertNotNull(city.getExpressionByName("cityaddress")); + assertNotNull(city.getExpressionByName("stateAndCountry")); + assertNotNull(city.getExpressionByName("cityaddress")); assertEquals(city.getExpressionByName("cityaddress").getDescription(), "city with state and city and zip"); assertEquals(city.getExpressionByName("cityaddress").getDisplayString(), "City Address"); @@ -412,7 +394,7 @@ public class TestCubeMetastoreClient { expr1 = new ExprSpec(); expr1.setExpr("contact(countrydim.name"); stateCountryExpr.addExpression(expr1); - Assert.fail("Expected add expression to fail because of syntax error"); + fail("Expected add expression to fail because of syntax error"); } catch (ParseException exc) { // Pass } @@ -424,13 +406,13 @@ public class TestCubeMetastoreClient { assertEquals(1, city.getExpressions().size()); ExprColumn stateAndCountryActual = city.getExpressionByName("stateAndCountry"); - Assert.assertNotNull(stateAndCountryActual.getExpressions()); + assertNotNull(stateAndCountryActual.getExpressions()); assertEquals(2, stateAndCountryActual.getExpressions().size()); - Assert.assertTrue(stateAndCountryActual.getExpressions().contains("concat(statedim.name, \"-\", countrydim.name)")); - Assert.assertTrue(stateAndCountryActual.getExpressions() + assertTrue(stateAndCountryActual.getExpressions().contains("concat(statedim.name, \"-\", countrydim.name)")); + assertTrue(stateAndCountryActual.getExpressions() .contains("concat(countrydim.name, \"-\", countrydim.name)")); - Assert.assertNotNull(city.getExpressionByName("stateAndCountry")); + assertNotNull(city.getExpressionByName("stateAndCountry")); assertEquals(city.getExpressionByName("stateAndCountry").getExpr(), "concat(statedim.name, \"-\", countrydim.name)"); @@ -441,7 +423,7 @@ public class TestCubeMetastoreClient { assertEquals(1, cityAltered.getExpressionByName("stateAndCountry").getExpressions().size()); - List<TableReference> chain = new ArrayList<TableReference>(); + List<TableReference> chain = new ArrayList<>(); chain.add(new TableReference("zipdim", "cityid")); chain.add(new TableReference("citydim", "id")); chain.add(new TableReference("citydim", "stateid")); @@ -458,8 +440,7 @@ public class TestCubeMetastoreClient { toAlter.alterAttribute(new ReferencedDimAtrribute(new FieldSchema("newRefDim", "int", "new ref-dim added"), "New city ref", new TableReference("citydim", "id"))); toAlter.alterAttribute(new BaseDimAttribute(new FieldSchema("f2", "varchar", "modified field"))); - List<TableReference> stateRefs = new ArrayList<TableReference>(); - stateRefs.add(new TableReference("statedim", "id")); + List<TableReference> stateRefs = Lists.newArrayList(new TableReference("statedim", "id")); toAlter.alterAttribute(new ReferencedDimAtrribute(new FieldSchema("stateid", "int", "state id"), "State refer", stateRefs)); toAlter.removeAttribute("f1"); @@ -473,32 +454,32 @@ public class TestCubeMetastoreClient { assertEquals(toAlter, altered); - Assert.assertNotNull(altered.getAttributeByName("newZipDim")); - Assert.assertNotNull(altered.getAttributeByName("newRefDim")); - Assert.assertNotNull(altered.getAttributeByName("f2")); - Assert.assertNotNull(altered.getAttributeByName("stateid")); - Assert.assertNull(altered.getAttributeByName("f1")); + assertNotNull(altered.getAttributeByName("newZipDim")); + assertNotNull(altered.getAttributeByName("newRefDim")); + assertNotNull(altered.getAttributeByName("f2")); + assertNotNull(altered.getAttributeByName("stateid")); + assertNull(altered.getAttributeByName("f1")); assertEquals(1, altered.getExpressions().size()); - Assert.assertNotNull(altered.getExpressionByName("formattedcode")); + assertNotNull(altered.getExpressionByName("formattedcode")); assertEquals(altered.getExpressionByName("formattedcode").getExpr(), "format_number(code, \"#,###,###\")"); CubeDimAttribute newzipdim = altered.getAttributeByName("newZipDim"); - Assert.assertTrue(newzipdim instanceof BaseDimAttribute); + assertTrue(newzipdim instanceof BaseDimAttribute); assertEquals(((BaseDimAttribute) newzipdim).getType(), "int"); assertEquals((((BaseDimAttribute) newzipdim).getNumOfDistinctValues().get()), Long.valueOf(1000)); CubeDimAttribute newrefdim = altered.getAttributeByName("newRefDim"); - Assert.assertTrue(newrefdim instanceof ReferencedDimAtrribute); + assertTrue(newrefdim instanceof ReferencedDimAtrribute); assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().size(), 1); assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().get(0).getDestTable(), cityDim.getName()); assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().get(0).getDestColumn(), "id"); CubeDimAttribute f2 = altered.getAttributeByName("f2"); - Assert.assertTrue(f2 instanceof BaseDimAttribute); + assertTrue(f2 instanceof BaseDimAttribute); assertEquals(((BaseDimAttribute) f2).getType(), "varchar"); CubeDimAttribute stateid = altered.getAttributeByName("stateid"); - Assert.assertTrue(stateid instanceof ReferencedDimAtrribute); + assertTrue(stateid instanceof ReferencedDimAtrribute); assertEquals(((ReferencedDimAtrribute) stateid).getReferences().size(), 1); assertEquals(((ReferencedDimAtrribute) stateid).getReferences().get(0).getDestTable(), stateDim.getName()); assertEquals(((ReferencedDimAtrribute) stateid).getReferences().get(0).getDestColumn(), "id"); @@ -521,34 +502,32 @@ public class TestCubeMetastoreClient { private void validateDim(Dimension udim, Set<CubeDimAttribute> attrs, String basedim, String referdim) throws HiveException { - Assert.assertTrue(client.tableExists(udim.getName())); + assertTrue(client.tableExists(udim.getName())); Table dimTbl = client.getHiveTable(udim.getName()); - Assert.assertTrue(client.isDimension(dimTbl)); + assertTrue(client.isDimension(dimTbl)); Dimension dim = new Dimension(dimTbl); - Assert.assertTrue(udim.equals(dim)); - Assert.assertTrue(udim.equals(client.getDimension(udim.getName()))); + assertTrue(udim.equals(dim)); + assertTrue(udim.equals(client.getDimension(udim.getName()))); assertEquals(dim.getAttributes().size(), attrs.size()); - Assert.assertNotNull(dim.getAttributeByName(basedim)); - Assert.assertTrue(dim.getAttributeByName(basedim) instanceof BaseDimAttribute); + assertNotNull(dim.getAttributeByName(basedim)); + assertTrue(dim.getAttributeByName(basedim) instanceof BaseDimAttribute); if (referdim != null) { - Assert.assertNotNull(dim.getAttributeByName(referdim)); - Assert.assertTrue(dim.getAttributeByName(referdim) instanceof ReferencedDimAtrribute); + assertNotNull(dim.getAttributeByName(referdim)); + assertTrue(dim.getAttributeByName(referdim) instanceof ReferencedDimAtrribute); } - assertEquals(udim.getAttributeNames().size() + udim.getExpressionNames().size(), dim.getAllFieldNames() - .size()); + assertEquals(udim.getAttributeNames().size() + udim.getExpressionNames().size(), dim.getAllFieldNames().size()); } @Test(priority = 1) public void testCube() throws Exception { - client.createCube(CUBE_NAME, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, - new HashMap<String, String>()); - Assert.assertTrue(client.tableExists(CUBE_NAME)); + client.createCube(CUBE_NAME, cubeMeasures, cubeDimensions, cubeExpressions, joinChains, emptyHashMap); + assertTrue(client.tableExists(CUBE_NAME)); Table cubeTbl = client.getHiveTable(CUBE_NAME); - Assert.assertTrue(client.isCube(cubeTbl)); + assertTrue(client.isCube(cubeTbl)); Cube cube2 = new Cube(cubeTbl); - Assert.assertTrue(cube.equals(cube2)); - Assert.assertFalse(cube2.isDerivedCube()); - Assert.assertTrue(cube2.getTimedDimensions().isEmpty()); + assertTrue(cube.equals(cube2)); + assertFalse(cube2.isDerivedCube()); + assertTrue(cube2.getTimedDimensions().isEmpty()); assertEquals(cubeMeasures.size(), cube2.getMeasureNames().size()); // +8 is for hierarchical dimension assertEquals(cubeDimensions.size() + 8, cube2.getDimAttributeNames().size()); @@ -558,32 +537,32 @@ public class TestCubeMetastoreClient { assertEquals(cubeDimensions.size(), cube2.getDimAttributes().size()); assertEquals(cubeDimensions.size() + 8 + cubeMeasures.size() + cubeExpressions.size(), cube2 .getAllFieldNames().size()); - Assert.assertNotNull(cube2.getMeasureByName("msr4")); + assertNotNull(cube2.getMeasureByName("msr4")); assertEquals(cube2.getMeasureByName("msr4").getDescription(), "fourth measure"); assertEquals(cube2.getMeasureByName("msr4").getDisplayString(), "Measure4"); - Assert.assertNotNull(cube2.getDimAttributeByName("location")); + assertNotNull(cube2.getDimAttributeByName("location")); assertEquals(cube2.getDimAttributeByName("location").getDescription(), "location hierarchy"); - Assert.assertNotNull(cube2.getDimAttributeByName("dim1")); + assertNotNull(cube2.getDimAttributeByName("dim1")); assertEquals(cube2.getDimAttributeByName("dim1").getDescription(), "basedim"); - Assert.assertNull(cube2.getDimAttributeByName("dim1").getDisplayString()); - Assert.assertNotNull(cube2.getDimAttributeByName("dim2")); + assertNull(cube2.getDimAttributeByName("dim1").getDisplayString()); + assertNotNull(cube2.getDimAttributeByName("dim2")); assertEquals(cube2.getDimAttributeByName("dim2").getDescription(), "ref dim"); assertEquals(cube2.getDimAttributeByName("dim2").getDisplayString(), "Dim2 refer"); - Assert.assertNotNull(cube2.getExpressionByName("msr5")); + assertNotNull(cube2.getExpressionByName("msr5")); assertEquals(cube2.getExpressionByName("msr5").getDescription(), "fifth measure"); assertEquals(cube2.getExpressionByName("msr5").getDisplayString(), "Avg msr5"); - Assert.assertNotNull(cube2.getExpressionByName("booleancut")); + assertNotNull(cube2.getExpressionByName("booleancut")); assertEquals(cube2.getExpressionByName("booleancut").getDescription(), "a boolean expression"); assertEquals(cube2.getExpressionByName("booleancut").getDisplayString(), "Boolean Cut"); assertEquals(cube2.getExpressionByName("booleancut").getExpressions().size(), 2); // Validate expression can contain delimiter character - List<String> booleanCutExprs = new ArrayList<String>(cube2.getExpressionByName("booleancut").getExpressions()); - Assert.assertTrue(booleanCutExprs.contains("dim1 | dim2 AND dim2 = 'XYZ'")); - Assert.assertTrue(cube2.allFieldsQueriable()); + List<String> booleanCutExprs = new ArrayList<>(cube2.getExpressionByName("booleancut").getExpressions()); + assertTrue(booleanCutExprs.contains("dim1 | dim2 AND dim2 = 'XYZ'")); + assertTrue(cube2.allFieldsQueriable()); - Assert.assertTrue(cube2.getJoinChainNames().contains("cityfromzip")); - Assert.assertTrue(cube2.getJoinChainNames().contains("city")); - Assert.assertFalse(cube2.getJoinChains().isEmpty()); + assertTrue(cube2.getJoinChainNames().contains("cityfromzip")); + assertTrue(cube2.getJoinChainNames().contains("city")); + assertFalse(cube2.getJoinChains().isEmpty()); assertEquals(cube2.getJoinChains().size(), 2); JoinChain zipchain = cube2.getChainByName("cityfromzip"); assertEquals(zipchain.getDisplayString(), "Zip City"); @@ -606,63 +585,63 @@ public class TestCubeMetastoreClient { assertEquals(citychain.getPaths().get(0).getReferences().size(), 2); assertEquals(citychain.getPaths().get(0).getReferences().get(0).toString(), "testmetastorecube.cityid"); assertEquals(citychain.getPaths().get(0).getReferences().get(1).toString(), "citydim.id"); - Assert.assertNotNull(cube2.getDimAttributeByName("zipcityname")); + assertNotNull(cube2.getDimAttributeByName("zipcityname")); ChainRefCol zipCityChain = ((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")) .getChainRefColumns().get(0); assertEquals(zipCityChain.getChainName(), "cityfromzip"); assertEquals(zipCityChain.getRefColumn(), "name"); - client.createDerivedCube(CUBE_NAME, DERIVED_CUBE_NAME, measures, dimensions, new HashMap<String, String>(), 0L); - Assert.assertTrue(client.tableExists(DERIVED_CUBE_NAME)); + client.createDerivedCube(CUBE_NAME, DERIVED_CUBE_NAME, measures, dimensions, emptyHashMap, 0L); + assertTrue(client.tableExists(DERIVED_CUBE_NAME)); Table derivedTbl = client.getHiveTable(DERIVED_CUBE_NAME); - Assert.assertTrue(client.isCube(derivedTbl)); + assertTrue(client.isCube(derivedTbl)); DerivedCube dcube2 = new DerivedCube(derivedTbl, cube); - Assert.assertTrue(derivedCube.equals(dcube2)); - Assert.assertTrue(dcube2.isDerivedCube()); - Assert.assertTrue(dcube2.getTimedDimensions().isEmpty()); + assertTrue(derivedCube.equals(dcube2)); + assertTrue(dcube2.isDerivedCube()); + assertTrue(dcube2.getTimedDimensions().isEmpty()); assertEquals(measures.size(), dcube2.getMeasureNames().size()); assertEquals(dimensions.size(), dcube2.getDimAttributeNames().size()); assertEquals(measures.size(), dcube2.getMeasures().size()); assertEquals(dimensions.size(), dcube2.getDimAttributes().size()); - Assert.assertNotNull(dcube2.getMeasureByName("msr3")); - Assert.assertNull(dcube2.getMeasureByName("msr4")); - Assert.assertNull(dcube2.getDimAttributeByName("location")); - Assert.assertNotNull(dcube2.getDimAttributeByName("dim1")); - Assert.assertTrue(dcube2.allFieldsQueriable()); + assertNotNull(dcube2.getMeasureByName("msr3")); + assertNull(dcube2.getMeasureByName("msr4")); + assertNull(dcube2.getDimAttributeByName("location")); + assertNotNull(dcube2.getDimAttributeByName("dim1")); + assertTrue(dcube2.allFieldsQueriable()); client.createCube(CUBE_NAME_WITH_PROPS, cubeMeasures, cubeDimensions, CUBE_PROPERTIES); - Assert.assertTrue(client.tableExists(CUBE_NAME_WITH_PROPS)); + assertTrue(client.tableExists(CUBE_NAME_WITH_PROPS)); cubeTbl = client.getHiveTable(CUBE_NAME_WITH_PROPS); - Assert.assertTrue(client.isCube(cubeTbl)); + assertTrue(client.isCube(cubeTbl)); cube2 = new Cube(cubeTbl); - Assert.assertTrue(cubeWithProps.equals(cube2)); - Assert.assertFalse(cube2.isDerivedCube()); - Assert.assertFalse(cubeWithProps.getTimedDimensions().isEmpty()); - Assert.assertTrue(cubeWithProps.getTimedDimensions().contains("dt")); - Assert.assertTrue(cubeWithProps.getTimedDimensions().contains("mydate")); + assertTrue(cubeWithProps.equals(cube2)); + assertFalse(cube2.isDerivedCube()); + assertFalse(cubeWithProps.getTimedDimensions().isEmpty()); + assertTrue(cubeWithProps.getTimedDimensions().contains("dt")); + assertTrue(cubeWithProps.getTimedDimensions().contains("mydate")); assertEquals(cubeMeasures.size(), cube2.getMeasureNames().size()); assertEquals(cubeDimensions.size() + 8, cube2.getDimAttributeNames().size()); assertEquals(cubeMeasures.size(), cube2.getMeasures().size()); assertEquals(cubeDimensions.size(), cube2.getDimAttributes().size()); - Assert.assertNotNull(cube2.getMeasureByName("msr4")); - Assert.assertNotNull(cube2.getDimAttributeByName("location")); - Assert.assertFalse(cube2.allFieldsQueriable()); + assertNotNull(cube2.getMeasureByName("msr4")); + assertNotNull(cube2.getDimAttributeByName("location")); + assertFalse(cube2.allFieldsQueriable()); client.createDerivedCube(CUBE_NAME_WITH_PROPS, DERIVED_CUBE_NAME_WITH_PROPS, measures, dimensions, CUBE_PROPERTIES, 0L); - Assert.assertTrue(client.tableExists(DERIVED_CUBE_NAME_WITH_PROPS)); + assertTrue(client.tableExists(DERIVED_CUBE_NAME_WITH_PROPS)); derivedTbl = client.getHiveTable(DERIVED_CUBE_NAME_WITH_PROPS); - Assert.assertTrue(client.isCube(derivedTbl)); + assertTrue(client.isCube(derivedTbl)); dcube2 = new DerivedCube(derivedTbl, cubeWithProps); - Assert.assertTrue(derivedCubeWithProps.equals(dcube2)); - Assert.assertTrue(dcube2.isDerivedCube()); - Assert.assertNotNull(derivedCubeWithProps.getProperties().get("cube.custom.prop")); + assertTrue(derivedCubeWithProps.equals(dcube2)); + assertTrue(dcube2.isDerivedCube()); + assertNotNull(derivedCubeWithProps.getProperties().get("cube.custom.prop")); assertEquals(derivedCubeWithProps.getProperties().get("cube.custom.prop"), "myval"); - Assert.assertNull(dcube2.getMeasureByName("msr4")); - Assert.assertNotNull(dcube2.getMeasureByName("msr3")); - Assert.assertNull(dcube2.getDimAttributeByName("location")); - Assert.assertNotNull(dcube2.getDimAttributeByName("dim1")); - Assert.assertTrue(dcube2.allFieldsQueriable()); + assertNull(dcube2.getMeasureByName("msr4")); + assertNotNull(dcube2.getMeasureByName("msr3")); + assertNull(dcube2.getDimAttributeByName("location")); + assertNotNull(dcube2.getDimAttributeByName("dim1")); + assertTrue(dcube2.allFieldsQueriable()); } @Test(priority = 1) @@ -683,23 +662,23 @@ public class TestCubeMetastoreClient { toAlter.removeTimedDimension("dt"); JoinChain cityChain = new JoinChain("city", "Cube City", "cube city desc modified"); - List<TableReference> chain = new ArrayList<TableReference>(); + List<TableReference> chain = new ArrayList<>(); chain.add(new TableReference(cubeName, "cityid")); chain.add(new TableReference("citydim", "id")); cityChain.addPath(chain); toAlter.alterJoinChain(cityChain); toAlter.removeJoinChain("cityFromZip"); - Assert.assertNotNull(toAlter.getMeasureByName("testAddMsr1")); - Assert.assertNotNull(toAlter.getMeasureByName("msr3")); + assertNotNull(toAlter.getMeasureByName("testAddMsr1")); + assertNotNull(toAlter.getMeasureByName("msr3")); assertEquals(toAlter.getMeasureByName("msr3").getDisplayString(), "Measure3Altered"); assertEquals(toAlter.getMeasureByName("msr3").getDescription(), "third altered measure"); - Assert.assertNull(toAlter.getMeasureByName("msr4")); - Assert.assertNotNull(toAlter.getDimAttributeByName("testAddDim1")); + assertNull(toAlter.getMeasureByName("msr4")); + assertNotNull(toAlter.getDimAttributeByName("testAddDim1")); assertEquals(toAlter.getDimAttributeByName("testAddDim1").getDescription(), "dim to add"); - Assert.assertNotNull(toAlter.getDimAttributeByName("dim1")); + assertNotNull(toAlter.getDimAttributeByName("dim1")); assertEquals(toAlter.getDimAttributeByName("dim1").getDescription(), "basedim altered"); - Assert.assertNull(toAlter.getDimAttributeByName("location2")); + assertNull(toAlter.getDimAttributeByName("location2")); client.alterCube(cubeName, toAlter); @@ -708,35 +687,35 @@ public class TestCubeMetastoreClient { Cube altered = new Cube(alteredHiveTbl); assertEquals(toAlter, altered); - Assert.assertNotNull(altered.getMeasureByName("testAddMsr1")); + assertNotNull(altered.getMeasureByName("testAddMsr1")); CubeMeasure addedMsr = altered.getMeasureByName("testAddMsr1"); assertEquals(addedMsr.getType(), "int"); - Assert.assertNotNull(altered.getDimAttributeByName("testAddDim1")); + assertNotNull(altered.getDimAttributeByName("testAddDim1")); BaseDimAttribute addedDim = (BaseDimAttribute) altered.getDimAttributeByName("testAddDim1"); assertEquals(addedDim.getType(), "string"); assertEquals(addedDim.getDescription(), "dim to add"); - Assert.assertTrue(altered.getTimedDimensions().contains("zt")); + assertTrue(altered.getTimedDimensions().contains("zt")); assertEquals(altered.getMeasureByName("msr3").getDisplayString(), "Measure3Altered"); assertEquals(altered.getMeasureByName("msr3").getDescription(), "third altered measure"); - Assert.assertNotNull(altered.getDimAttributeByName("dim1")); + assertNotNull(altered.getDimAttributeByName("dim1")); assertEquals(altered.getDimAttributeByName("dim1").getDescription(), "basedim altered"); - Assert.assertNull(altered.getDimAttributeByName("location2")); - Assert.assertNull(altered.getChainByName("cityFromZip")); + assertNull(altered.getDimAttributeByName("location2")); + assertNull(altered.getChainByName("cityFromZip")); assertEquals(altered.getChainByName("city").getDescription(), "cube city desc modified"); toAlter.alterMeasure(new ColumnMeasure(new FieldSchema("testAddMsr1", "double", "testAddMeasure"))); client.alterCube(cubeName, toAlter); altered = new Cube(Hive.get(conf).getTable(cubeName)); addedMsr = altered.getMeasureByName("testaddmsr1"); - Assert.assertNotNull(addedMsr); + assertNotNull(addedMsr); assertEquals(addedMsr.getType(), "double"); - Assert.assertTrue(client.getAllFacts(altered).isEmpty()); + assertTrue(client.getAllFacts(altered).isEmpty()); } @Test(priority = 2) public void testAlterDerivedCube() throws Exception { String name = "alter_derived_cube"; - client.createDerivedCube(CUBE_NAME, name, measures, dimensions, new HashMap<String, String>(), 0L); + client.createDerivedCube(CUBE_NAME, name, measures, dimensions, emptyHashMap, 0L); // Test alter cube Table cubeTbl = client.getHiveTable(name); DerivedCube toAlter = new DerivedCube(cubeTbl, (Cube) client.getCube(CUBE_NAME)); @@ -745,34 +724,34 @@ public class TestCubeMetastoreClient { toAlter.addDimension("dim1StartTime"); toAlter.removeDimension("dim1"); - Assert.assertNotNull(toAlter.getMeasureByName("msr4")); - Assert.assertNotNull(toAlter.getMeasureByName("msr2")); - Assert.assertNull(toAlter.getMeasureByName("msr3")); - Assert.assertNotNull(toAlter.getDimAttributeByName("dim1StartTime")); - Assert.assertNotNull(toAlter.getDimAttributeByName("dim2")); - Assert.assertNull(toAlter.getDimAttributeByName("dim1")); + assertNotNull(toAlter.getMeasureByName("msr4")); + assertNotNull(toAlter.getMeasureByName("msr2")); + assertNull(toAlter.getMeasureByName("msr3")); + assertNotNull(toAlter.getDimAttributeByName("dim1StartTime")); + assertNotNull(toAlter.getDimAttributeByName("dim2")); + assertNull(toAlter.getDimAttributeByName("dim1")); client.alterCube(name, toAlter); DerivedCube altered = (DerivedCube) client.getCube(name); assertEquals(toAlter, altered); - Assert.assertNotNull(altered.getMeasureByName("msr4")); + assertNotNull(altered.getMeasureByName("msr4")); CubeMeasure addedMsr = altered.getMeasureByName("msr4"); assertEquals(addedMsr.getType(), "bigint"); - Assert.assertNotNull(altered.getDimAttributeByName("dim1StartTime")); + assertNotNull(altered.getDimAttributeByName("dim1StartTime")); BaseDimAttribute addedDim = (BaseDimAttribute) altered.getDimAttributeByName("dim1StartTime"); assertEquals(addedDim.getType(), "string"); - Assert.assertNotNull(addedDim.getStartTime()); + assertNotNull(addedDim.getStartTime()); client.dropCube(name); - Assert.assertFalse(client.tableExists(name)); + assertFalse(client.tableExists(name)); } @Test(priority = 2) public void testCubeFact() throws Exception { String factName = "testMetastoreFact"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size()); + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); for (CubeMeasure measure : cubeMeasures) { factColumns.add(measure.getColumn()); } @@ -780,114 +759,86 @@ public class TestCubeMetastoreClient { // add one dimension of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); - Map<String, Set<UpdatePeriod>> updatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); - List<String> timePartCols = new ArrayList<String>(); - partCols.add(getDatePartition()); - timePartCols.add(getDatePartitionKey()); - updates.add(HOURLY); - updates.add(DAILY); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - updatePeriods.put(c1, updates); - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + datePartSingleton, datePartKeySingleton); + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, Sets.newHashSet(HOURLY, DAILY)); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods); // create cube fact client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables); - Assert.assertTrue(client.tableExists(factName)); + assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); - Assert.assertTrue(client.isFactTable(cubeTbl)); - Assert.assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); + assertTrue(client.isFactTable(cubeTbl)); + assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME)); assertEquals(client.getAllFacts(client.getCube(CUBE_NAME)).get(0).getName(), factName.toLowerCase()); - assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), - factName.toLowerCase()); + assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(), factName.toLowerCase()); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); - Assert.assertTrue(cubeFact.equals(cubeFact2)); + assertTrue(cubeFact.equals(cubeFact2)); // Assert for storage tables for (String entry : storageTables.keySet()) { - String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry); - Assert.assertTrue(client.tableExists(storageTableName)); + String storageTableName = getFactOrDimtableStorageTableName(factName, entry); + assertTrue(client.tableExists(storageTableName)); } - Map<String, Date> timeParts = new HashMap<String, Date>(); - timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now); - timeParts.put("non_existing_part_col", now); + Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0, "non_existing_part_col", 0); // test error on adding invalid partition // test partition StoragePartitionDesc partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); - try{ + try { client.addPartition(partSpec, c1); fail("Add should fail since non_existing_part_col is non-existing"); - } catch(LensException e){ + } catch (LensException e) { assertEquals(e.getErrorCode(), LensCubeErrorCode.TIMELINE_ABSENT.getLensErrorInfo().getErrorCode()); } timeParts.remove("non_existing_part_col"); partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); client.addPartition(partSpec, c1); - Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, - new HashMap<String, String>())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); + assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); // Partition with different schema FieldSchema newcol = new FieldSchema("newcol", "int", "new col for part"); cubeFact.alterColumn(newcol); client.alterCubeFactTable(cubeFact.getName(), cubeFact, storageTables); - String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c1); + String storageTableName = getFactOrDimtableStorageTableName(factName, c1); assertEquals(client.getAllParts(storageTableName).size(), 1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); - Map<String, Date> timeParts2 = new HashMap<String, Date>(); - timeParts2.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus1); - StoragePartitionDesc partSpec2 = - new StoragePartitionDesc(cubeFact.getName(), timeParts2, null, HOURLY); + Map<String, Date> timeParts2 = getTimePartitionByOffsets(getDatePartitionKey(), 1); + StoragePartitionDesc partSpec2 = new StoragePartitionDesc(cubeFact.getName(), timeParts2, null, HOURLY); partSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); partSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); client.addPartition(partSpec2, c1); assertEquals(client.getAllParts(storageTableName).size(), 2); - Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, - new HashMap<String, String>())); - Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, - new HashMap<String, String>())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); + assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); + assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); client.dropPartition(cubeFact.getName(), c1, timeParts2, null, HOURLY); assertEquals(client.getAllParts(storageTableName).size(), 1); - Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, - new HashMap<String, String>())); - Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, - new HashMap<String, String>())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); + assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); + assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); client.dropPartition(cubeFact.getName(), c1, timeParts, null, HOURLY); assertEquals(client.getAllParts(storageTableName).size(), 0); - Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, - new HashMap<String, String>())); - Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, - new HashMap<String, String>())); - Assert.assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); + assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); + assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts2, emptyHashMap)); + assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); } @Test(priority = 2) public void testAlterCubeFact() throws Exception { String factName = "test_alter_fact"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size()); + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); for (CubeMeasure measure : cubeMeasures) { factColumns.add(measure.getColumn()); } @@ -895,29 +846,10 @@ public class TestCubeMetastoreClient { // add one dimension of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); - Map<String, Set<UpdatePeriod>> updatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - updates.add(HOURLY); - updates.add(DAILY); - - ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); - List<String> timePartCols = new ArrayList<String>(); - partCols.add(getDatePartition()); - timePartCols.add(getDatePartitionKey()); - updates.add(HOURLY); - updates.add(DAILY); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - updatePeriods.put(c1, updates); - - updatePeriods.put(c2, updates); - - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - storageTables.put(c2, s1); + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + datePartSingleton, datePartKeySingleton); + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, hourlyAndDaily, c2, hourlyAndDaily); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1, c2, s1); // create cube fact client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables); @@ -926,15 +858,11 @@ public class TestCubeMetastoreClient { factTable.alterColumn(new FieldSchema("testFactColAdd", "int", "test add column")); factTable.alterColumn(new FieldSchema("msr3", "int", "test alter column")); factTable.alterWeight(100L); - Map<String, String> newProp = new HashMap<String, String>(); - newProp.put("new.prop", "val"); + Map<String, String> newProp = getHashMap("new.prop", "val"); factTable.addProperties(newProp); factTable.addUpdatePeriod(c1, MONTHLY); factTable.removeUpdatePeriod(c1, HOURLY); - Set<UpdatePeriod> alterupdates = new HashSet<UpdatePeriod>(); - alterupdates.add(HOURLY); - alterupdates.add(DAILY); - alterupdates.add(MONTHLY); + Set<UpdatePeriod> alterupdates = Sets.newHashSet(HOURLY, DAILY, MONTHLY); factTable.alterStorage(c2, alterupdates); client.alterCubeFactTable(factName, factTable, storageTables); @@ -942,14 +870,14 @@ public class TestCubeMetastoreClient { Table factHiveTable = Hive.get(conf).getTable(factName); CubeFactTable altered = new CubeFactTable(factHiveTable); - Assert.assertTrue(altered.weight() == 100L); - Assert.assertTrue(altered.getProperties().get("new.prop").equals("val")); - Assert.assertTrue(altered.getUpdatePeriods().get(c1).contains(MONTHLY)); - Assert.assertFalse(altered.getUpdatePeriods().get(c1).contains(HOURLY)); - Assert.assertTrue(altered.getUpdatePeriods().get(c2).contains(MONTHLY)); - Assert.assertTrue(altered.getUpdatePeriods().get(c2).contains(DAILY)); - Assert.assertTrue(altered.getUpdatePeriods().get(c2).contains(HOURLY)); - Assert.assertTrue(altered.getCubeName().equalsIgnoreCase(CUBE_NAME.toLowerCase())); + assertTrue(altered.weight() == 100L); + assertTrue(altered.getProperties().get("new.prop").equals("val")); + assertTrue(altered.getUpdatePeriods().get(c1).contains(MONTHLY)); + assertFalse(altered.getUpdatePeriods().get(c1).contains(HOURLY)); + assertTrue(altered.getUpdatePeriods().get(c2).contains(MONTHLY)); + assertTrue(altered.getUpdatePeriods().get(c2).contains(DAILY)); + assertTrue(altered.getUpdatePeriods().get(c2).contains(HOURLY)); + assertTrue(altered.getCubeName().equalsIgnoreCase(CUBE_NAME.toLowerCase())); boolean contains = false; for (FieldSchema column : altered.getColumns()) { if (column.getName().equals("testfactcoladd") && column.getType().equals("int")) { @@ -957,28 +885,23 @@ public class TestCubeMetastoreClient { break; } } - Assert.assertTrue(contains); + assertTrue(contains); // alter storage table desc - String c1TableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c1); + String c1TableName = getFactOrDimtableStorageTableName(factName, c1); Table c1Table = client.getTable(c1TableName); - assertEquals(c1Table.getInputFormatClass().getCanonicalName(), - TextInputFormat.class.getCanonicalName()); - s1 = new StorageTableDesc(); - s1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); + assertEquals(c1Table.getInputFormatClass().getCanonicalName(), TextInputFormat.class.getCanonicalName()); + s1 = new StorageTableDesc(SequenceFileInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + datePartSingleton, datePartKeySingleton); s1.setFieldDelim(":"); storageTables.put(c1, s1); storageTables.put(c4, s1); - factTable.addStorage(c4, updates); + factTable.addStorage(c4, hourlyAndDaily); client.alterCubeFactTable(factName, factTable, storageTables); CubeFactTable altered2 = client.getCubeFact(factName); - Assert.assertTrue(client.tableExists(c1TableName)); + assertTrue(client.tableExists(c1TableName)); Table alteredC1Table = client.getTable(c1TableName); - assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(), - SequenceFileInputFormat.class.getCanonicalName()); + assertEquals(alteredC1Table.getInputFormatClass(), SequenceFileInputFormat.class); assertEquals(alteredC1Table.getSerdeParam(serdeConstants.FIELD_DELIM), ":"); boolean storageTableColsAltered = false; @@ -988,51 +911,51 @@ public class TestCubeMetastoreClient { break; } } - Assert.assertTrue(storageTableColsAltered); + assertTrue(storageTableColsAltered); - Assert.assertTrue(altered2.getStorages().contains("C4")); - Assert.assertTrue(altered2.getUpdatePeriods().get("C4").equals(updates)); - String c4TableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c4); - Assert.assertTrue(client.tableExists(c4TableName)); + assertTrue(altered2.getStorages().contains("C4")); + assertTrue(altered2.getUpdatePeriods().get("C4").equals(hourlyAndDaily)); + String c4TableName = getFactOrDimtableStorageTableName(factName, c4); + assertTrue(client.tableExists(c4TableName)); // add storage - client.addStorage(altered2, c3, updates, s1); + client.addStorage(altered2, c3, hourlyAndDaily, s1); CubeFactTable altered3 = client.getCubeFact(factName); - Assert.assertTrue(altered3.getStorages().contains("C3")); - Assert.assertTrue(altered3.getUpdatePeriods().get("C3").equals(updates)); - String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c3); - Assert.assertTrue(client.tableExists(storageTableName)); + assertTrue(altered3.getStorages().contains("C3")); + assertTrue(altered3.getUpdatePeriods().get("C3").equals(hourlyAndDaily)); + String storageTableName = getFactOrDimtableStorageTableName(factName, c3); + assertTrue(client.tableExists(storageTableName)); client.dropStorageFromFact(factName, c2); - storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c2); - Assert.assertFalse(client.tableExists(storageTableName)); + storageTableName = getFactOrDimtableStorageTableName(factName, c2); + assertFalse(client.tableExists(storageTableName)); List<CubeFactTable> cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME)); - List<String> cubeFactNames = new ArrayList<String>(); + List<String> cubeFactNames = new ArrayList<>(); for (CubeFactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } - Assert.assertTrue(cubeFactNames.contains(factName.toLowerCase())); + assertTrue(cubeFactNames.contains(factName.toLowerCase())); cubeFacts = client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)); - cubeFactNames = new ArrayList<String>(); + cubeFactNames = new ArrayList<>(); for (CubeFactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } - Assert.assertTrue(cubeFactNames.contains(factName.toLowerCase())); + assertTrue(cubeFactNames.contains(factName.toLowerCase())); client.dropFact(factName, true); - Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(factName, c1))); - Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(factName, c3))); - Assert.assertFalse(client.tableExists(factName)); + assertFalse(client.tableExists(getFactOrDimtableStorageTableName(factName, c1))); + assertFalse(client.tableExists(getFactOrDimtableStorageTableName(factName, c3))); + assertFalse(client.tableExists(factName)); cubeFacts = client.getAllFacts(cube); - cubeFactNames = new ArrayList<String>(); + cubeFactNames = new ArrayList<>(); for (CubeFactTable cfact : cubeFacts) { cubeFactNames.add(cfact.getName()); } - Assert.assertFalse(cubeFactNames.contains(factName.toLowerCase())); + assertFalse(cubeFactNames.contains(factName.toLowerCase())); } @Test(priority = 2) public void testCubeFactWithTwoTimedParts() throws Exception { String factName = "testMetastoreFactTimedParts"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size()); + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); for (CubeMeasure measure : cubeMeasures) { factColumns.add(measure.getColumn()); } @@ -1040,61 +963,40 @@ public class TestCubeMetastoreClient { // add one dimension of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); - Map<String, Set<UpdatePeriod>> updatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - updates.add(HOURLY); - updates.add(DAILY); + Set<UpdatePeriod> updates = Sets.newHashSet(HOURLY, DAILY); FieldSchema testDtPart = new FieldSchema("mydate", "string", "date part"); - ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); - List<String> timePartCols = new ArrayList<String>(); - partCols.add(getDatePartition()); - partCols.add(testDtPart); - timePartCols.add(getDatePartitionKey()); - timePartCols.add(testDtPart.getName()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - updatePeriods.put(c1, updates); - - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + Lists.newArrayList(getDatePartition(), testDtPart), + Lists.newArrayList(getDatePartitionKey(), testDtPart.getName())); + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, updates); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1); CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME_WITH_PROPS, factName, factColumns, updatePeriods); // create cube fact client.createCubeFactTable(CUBE_NAME_WITH_PROPS, factName, factColumns, updatePeriods, 0L, null, storageTables); - Assert.assertTrue(client.tableExists(factName)); + assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); - Assert.assertTrue(client.isFactTable(cubeTbl)); - Assert.assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME_WITH_PROPS)); + assertTrue(client.isFactTable(cubeTbl)); + assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME_WITH_PROPS)); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); - Assert.assertTrue(cubeFact.equals(cubeFact2)); + assertTrue(cubeFact.equals(cubeFact2)); // Assert for storage tables for (String entry : storageTables.keySet()) { - String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry); - Assert.assertTrue(client.tableExists(storageTableName)); + String storageTableName = getFactOrDimtableStorageTableName(factName, entry); + assertTrue(client.tableExists(storageTableName)); } - // test partition - Calendar cal = new GregorianCalendar(); - cal.setTime(now); - cal.add(Calendar.HOUR, -1); - Date testDt = cal.getTime(); - Map<String, Date> timeParts = new HashMap<String, Date>(); - timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now); - timeParts.put(testDtPart.getName(), testDt); + //test partition + Map<String, Date> timeParts = getTimePartitionByOffsets(getDatePartitionKey(), 0, testDtPart.getName(), -1); StoragePartitionDesc partSpec = new StoragePartitionDesc(cubeFact.getName(), timeParts, null, HOURLY); client.addPartition(partSpec, c1); - Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, - new HashMap<String, String>())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c1); + assertTrue(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + String storageTableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c1); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), 0); @@ -1104,81 +1006,61 @@ public class TestCubeMetastoreClient { client.dropPartition(cubeFact.getName(), c1, timeParts, null, HOURLY); assertEquals(client.getAllParts(storageTableName).size(), 0); - Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, - new HashMap<String, String>())); - Assert.assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - Assert.assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName())); + assertFalse(client.factPartitionExists(cubeFact.getName(), c1, HOURLY, timeParts, emptyHashMap)); + assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName())); } @Test(priority = 2) public void testCubeFactWithThreeTimedParts() throws Exception { String factName = "testMetastoreFact3TimedParts"; - List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size()); + List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size()); for (CubeMeasure measure : cubeMeasures) { factColumns.add(measure.getColumn()); } // add one dimension of the cube factColumns.add(new FieldSchema("zipcode", "int", "zip")); - - Map<String, Set<UpdatePeriod>> updatePeriods = new HashMap<String, Set<UpdatePeriod>>(); - Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>(); - updates.add(HOURLY); - updates.add(DAILY); FieldSchema itPart = new FieldSchema("it", "string", "date part"); FieldSchema etPart = new FieldSchema("et", "string", "date part"); - ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>(); - List<String> timePartCols = new ArrayList<String>(); - partCols.add(getDatePartition()); - partCols.add(itPart); - partCols.add(etPart); - timePartCols.add(getDatePartitionKey()); - timePartCols.add(itPart.getName()); - timePartCols.add(etPart.getName()); - StorageTableDesc s1 = new StorageTableDesc(); - s1.setInputFormat(TextInputFormat.class.getCanonicalName()); - s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName()); - s1.setPartCols(partCols); - s1.setTimePartCols(timePartCols); - updatePeriods.put(c1, updates); - updatePeriods.put(c2, updates); - - Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>(); - storageTables.put(c1, s1); - storageTables.put(c2, s1); + StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class, + Lists.newArrayList(getDatePartition(), itPart, etPart), + Lists.newArrayList(getDatePartitionKey(), itPart.getName(), etPart.getName())); + + Map<String, Set<UpdatePeriod>> updatePeriods = getHashMap(c1, hourlyAndDaily, c2, hourlyAndDaily); + Map<String, StorageTableDesc> storageTables = getHashMap(c1, s1, c2, s1); CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME_WITH_PROPS, factName, factColumns, updatePeriods); // create cube fact client.createCubeFactTable(CUBE_NAME_WITH_PROPS, factName, factColumns, updatePeriods, 0L, null, storageTables); - Assert.assertTrue(client.tableExists(factName)); + assertTrue(client.tableExists(factName)); Table cubeTbl = client.getHiveTable(factName); - Assert.assertTrue(client.isFactTable(cubeTbl)); - Assert.assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME_WITH_PROPS)); + assertTrue(client.isFactTable(cubeTbl)); + assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME_WITH_PROPS)); CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl); - Assert.assertTrue(cubeFact.equals(cubeFact2)); + assertTrue(cubeFact.equals(cubeFact2)); String[] storages = new String[]{c1, c2}; String[] partColNames = new String[]{getDatePartitionKey(), itPart.getName(), etPart.getName()}; // Assert for storage tables for (String entry : storageTables.keySet()) { - String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry); - Assert.assertTrue(client.tableExists(storageTableName)); + String storageTableName = getFactOrDimtableStorageTableName(factName, entry); + assertTrue(client.tableExists(storageTableName)); } - String c1TableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c1); - String c2TableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c2); + String c1TableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c1); + String c2TableName = getFactOrDimtableStorageTableName(cubeFact.getName(), c2); - Table c1Table = client.getHiveTable(c1TableName); + client.getHiveTable(c1TableName); Table c2Table = client.getHiveTable(c2TableName); - c2Table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, + c2Table.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, getDatePartitionKey()), StoreAllPartitionTimeline.class.getCanonicalName()); - c2Table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, + c2Table.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, itPart.getName()), StoreAllPartitionTimeline.class.getCanonicalName()); - c2Table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, + c2Table.getParameters().put(getPartitionTimelineStorageClassKey(HOURLY, etPart.getName()), StoreAllPartitionTimeline.class.getCanonicalName()); client.pushHiveTable(c2Table); @@ -1197,33 +1079,23 @@ public class TestCubeMetastoreClient { StoreAllPartitionTimeline timelineEtC2 = ((StoreAllPartitionTimeline) client.partitionTimelineCache.get( factName, c2, HOURLY, etPart.getName())); - Map<String, Date> timeParts1 = new HashMap<String, Date>(); - timeParts1.put(TestCubeMetastoreClient.getDatePartitionKey(), now); - timeParts1.put(itPart.getName(), now); - timeParts1.put(etPart.getName(), now); - StoragePartitionDesc partSpec1 = new StoragePartitionDesc(cubeFact.getName(), timeParts1, null, - HOURLY); + Map<String, Date> timeParts1 = getTimePartitionByOffsets(getDatePartitionKey(), 0, itPart.getName(), 0, + etPart.getName(), 0); + StoragePartitionDesc partSpec1 = new StoragePartitionDesc(cubeFact.getName(), timeParts1, null, HOURLY); - Map<String, Date> timeParts2 = new HashMap<String, Date>(); - timeParts2.put(TestCubeMetastoreClient.getDatePartitionKey(), now); - timeParts2.put(etPart.getName(), nowPlus1); - Map<String, String> nonTimeSpec = new HashMap<String, String>(); - nonTimeSpec.put(itPart.getName(), "default"); + Map<String, Date> timeParts2 = getTimePartitionByOffsets(getDatePartitionKey(), 0, etPart.getName(), 1); + Map<String, String> nonTimeSpec = getHashMap(itPart.getName(), "default"); final StoragePartitionDesc partSpec2 = new StoragePartitionDesc(cubeFact.getName(), timeParts2, nonTimeSpec, HOURLY); - Map<String, Date> timeParts3 = new HashMap<String, Date>(); - timeParts3.put(TestCubeMetastoreClient.getDatePartitionKey(), now); - timeParts3.put(etPart.getName(), now); + Map<String, Date> timeParts3 = getTimePartitionByOffsets(getDatePartitionKey(), 0, etPart.getName(), 0); final StoragePartitionDesc partSpec3 = new StoragePartitionDesc(cubeFact.getName(), timeParts3, nonTimeSpec, HOURLY); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1); client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c2); - PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, - getDatePartitionKey()); - PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, - getDatePartitionKey()); + PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); + PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey()); assertEquals(timeline1Temp.getClass(), EndsAndHolesPartitionTimeline.class); assertEquals(timeline2Temp.getClass(), StoreAllPartitionTimeline.class); @@ -1233,32 +1105,25 @@ public class TestCubeMetastoreClient { assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, now, now); - assertTimeline(timelineEt, timelineEtC2, HOURLY, now, nowPlus1); - assertTimeline(timelineIt, timelineItC2, HOURLY, now, now); + assertTimeline(timelineDt, timelineDtC2, HOURLY, 0, 0); + assertTimeline(timelineEt, timelineEtC2, HOURLY, 0, 1); + assertTimeline(timelineIt, timelineItC2, HOURLY, 0, 0); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c2, etPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c2, etPart.getName())); assertNoPartitionNamedLatest(c1TableName, partColNames); assertNoPartitionNamedLatest(c2TableName, partColNames); - Map<String, Date> timeParts4 = new HashMap<String, Date>(); - timeParts4.put(TestCubeMetastoreClient.getDatePartitionKey(), now); - timeParts4.put(itPart.getName(), nowPlus1); - timeParts4.put(etPart.getName(), nowMinus1); - final StoragePartitionDesc partSpec4 = new StoragePartitionDesc(cubeFact.getName(), timeParts4, null, - HOURLY); + Map<String, Date> timeParts4 = getTimePartitionByOffsets(getDatePartitionKey(), 0, itPart.getName(), 1, + etPart.getName(), -1); + final StoragePartitionDesc partSpec4 = new StoragePartitionDesc(cubeFact.getName(), timeParts4, null, HOURLY); - Map<String, Date> timeParts5 = new HashMap<String, Date>(); - timeParts5.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus1); - timeParts5.put(itPart.getName(), nowMinus1); - timeParts5.put(etPart.getName(), nowMinus2); - final StoragePartitionDesc partSpec5 = new StoragePartitionDesc(cubeFact.getName(), timeParts5, null, - HOURLY); + Map<String, Date> timeParts5 = getTimePartitionByOffsets(getDatePartitionKey(), 1, itPart.getName(), -1, + etPart.getName(), -2); + final StoragePartitionDesc partSpec5 = new StoragePartitionDesc(cubeFact.getName(), timeParts5, null, HOURLY); client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1); client.addPartitions(Arrays.asList(partSpec4, partSpec5), c2); @@ -1268,21 +1133,17 @@ public class TestCubeMetastoreClient { assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, now, nowPlus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus2, nowPlus1); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus1, nowPlus1); + assertTimeline(timelineDt, timelineDtC2, HOURLY, 0, 1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -2, 1); + assertTimeline(timelineIt, timelineItC2, HOURLY, -1, 1); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); - Map<String, Date> timeParts6 = new HashMap<String, Date>(); - timeParts6.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus2); - timeParts6.put(itPart.getName(), nowMinus1); - timeParts6.put(etPart.getName(), nowMinus2); - final StoragePartitionDesc partSpec6 = new StoragePartitionDesc(cubeFact.getName(), timeParts6, null, - HOURLY); + Map<String, Date> timeParts6 = getTimePartitionByOffsets(getDatePartitionKey(), -2, itPart.getName(), -1, + etPart.getName(), -2); + final StoragePartitionDesc partSpec6 = new StoragePartitionDesc(cubeFact.getName(), timeParts6, null, HOURLY); client.addPartition(partSpec6, c1); client.addPartition(partSpec6, c2); @@ -1292,21 +1153,17 @@ public class TestCubeMetastoreClient { assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus2, nowPlus1, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus2, nowPlus1); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus1, nowPlus1); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -2, 1, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -2, 1); + assertTimeline(timelineIt, timelineItC2, HOURLY, -1, 1); - Map<String, Date> timeParts7 = new HashMap<String, Date>(); - timeParts7.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus5); - timeParts7.put(itPart.getName(), nowMinus5); - timeParts7.put(etPart.getName(), nowMinus5); - final StoragePartitionDesc partSpec7 = new StoragePartitionDesc(cubeFact.getName(), timeParts7, null, - HOURLY); + Map<String, Date> timeParts7 = getTimePartitionByOffsets(getDatePartitionKey(), -5, itPart.getName(), -5, + etPart.getName(), -5); + final StoragePartitionDesc partSpec7 = new StoragePartitionDesc(cubeFact.getName(), timeParts7, null, HOURLY); client.addPartition(partSpec7, c1); client.addPartition(partSpec7, c2); - List<Partition> c1Parts = client.getAllParts(c1TableName); List<Partition> c2Parts = client.getAllParts(c2TableName); @@ -1330,70 +1187,67 @@ public class TestCubeMetastoreClient { assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3, nowMinus2); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -5, 1, -4, -3, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -5, 1, -4, -3); + assertTimeline(timelineIt, timelineItC2, HOURLY, -5, 1, -4, -3, -2); assertNoPartitionNamedLatest(c1TableName, partColNames); assertSameTimelines(factName, storages, HOURLY, partColNames); assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get( - MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); + getPartitionTimelineCachePresenceKey()), "true"); assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get( - MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); + getPartitionTimelineCachePresenceKey()), "true"); // alter tables and see timeline still exists client.alterCubeFactTable(factName, cubeFact, storageTables); assertSameTimelines(factName, storages, HOURLY, partColNames); assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get( - MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); + getPartitionTimelineCachePresenceKey()), "true"); assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get( - MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); + getPartitionTimelineCachePresenceKey()), "true"); client.dropPartition(cubeFact.getName(), c1, timeParts5, null, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts5, null, HOURLY); assertEquals(client.getAllParts(c1TableName).size(), 6); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); assertNoPartitionNamedLatest(c1TableName, partColNames); assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus5, now, nowMinus4, nowMinus3, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3, nowMinus2); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -5, 0, -4, -3, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -5, 1, -4, -3); + assertTimeline(timelineIt, timelineItC2, HOURLY, -5, 1, -4, -3, -2); client.dropPartition(cubeFact.getName(), c1, timeParts7, null, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts7, null, HOURLY); assertEquals(client.getAllParts(c1TableName).size(), 5); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); assertNoPartitionNamedLatest(c1TableName, partColNames); assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus2, now, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus2, nowPlus1); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus1, nowPlus1); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -2, 0, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -2, 1); + assertTimeline(timelineIt, timelineItC2, HOURLY, -1, 1); client.dropPartition(cubeFact.getName(), c1, timeParts2, nonTimeSpec, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts2, nonTimeSpec, HOURLY); assertEquals(client.getAllParts(c1TableName).size(), 4); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, - TestCubeMetastoreClient.getDatePartitionKey())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); - Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, getDatePartitionKey())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName())); + assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, etPart.getName())); assertNoPartitionNamedLatest(c1TableName, partColNames); assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus2, now, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus2, now); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus1, nowPlus1); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -2, 0, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -2, 0); + assertTimeline(timelineIt, timelineItC2, HOURLY, -1, 1); client.dropPartition(cubeFact.getName(), c1, timeParts4, null, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts4, null, HOURLY); @@ -1401,28 +1255,28 @@ public class TestCubeMetastoreClient { assertNoPartitionNamedLatest(c1TableName, partColNames); assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus2, now, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus2, now, nowMinus1); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus1, now); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -2, 0, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -2, 0, -1); + assertTimeline(timelineIt, timelineItC2, HOURLY, -1, 0); client.dropPartition(cubeFact.getName(), c1, timeParts3, nonTimeSpec, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts3, nonTimeSpec, HOURLY); assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, nowMinus2, now, nowMinus1); - assertTimeline(timelineEt, timelineEtC2, HOURLY, nowMinus2, now, nowMinus1); - assertTimeline(timelineIt, timelineItC2, HOURLY, nowMinus1, now); + assertTimeline(timelineDt, timelineDtC2, HOURLY, -2, 0, -1); + assertTimeline(timelineEt, timelineEtC2, HOURLY, -2, 0, -1); + assertTimeline(timelineIt, timelineItC2, HOURLY, -1, 0); client.dropPartition(cubeFact.getName(), c1, timeParts6, null, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts6, null, HOURLY); assertSameTimelines(factName, storages, HOURLY, partColNames); - assertTimeline(timelineDt, timelineDtC2, HOURLY, now, now); - assertTimeline(timelineEt, timelineEtC2, HOURLY, now, now); - assertTimeline(timelineIt, timelineItC2, HOURLY, now, now); + assertTimeline(timelineDt, timelineDtC2, HOURLY, 0, 0); + assertTimeline(timelineEt, timelineEtC2, HOURLY, 0, 0); + assertTimeline(timelineIt, timelineItC2, HOURLY, 0, 0); client.dropPartition(cubeFact.getName(), c1, timeParts1, null, HOURLY); client.dropPartition(cubeFact.getName(), c2, timeParts1, null, HOURLY); assertSameTimelines(factName, storages, HOURLY, partColNames); - Assert.assertTrue(timelineDt.isEmpty()); - Assert.assertTrue(timelineEt.isEmpty()); - Assert.assertTrue(timelineIt.isEmpty()); + assertTrue(timelineDt.isEmpty()); + assertTrue(timelineEt.isEmpty()); + assertTrue(timelineIt.isEmpty()); } @@ -1438,7 +1292,16 @@ public class TestCubeMetastoreClient { "complete name differs at element " + i); } } - + private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, + StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, + int firstOffset, int latestOffset, int... holeOffsets) throws LensException { + Date[] holeDates = new Date[holeOffsets.length]; + for(int i = 0; i < holeOffsets.length; i++) { + holeDates[i] = getDateWithOffset(holeOffsets[i]); + } + assertTimeline(endsAndHolesPartitionTimeline, storeAllPartitionTimeline, updatePeriod, + getDateWithOffset(firstOffset), getDateWithOffset(latestOffset), holeDates); + } private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, Date first, Date latest, Date... holes) throws LensException { @@ -1448,10 +1311,10
<TRUNCATED>
