Repository: incubator-lens Updated Branches: refs/heads/master 85b3d9e9e -> 76218cb14
LENS-281: Timeline should have same contract in toProperties() and initFromProperties() (Rajat Khandelwal via amareshwari) Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/76218cb1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/76218cb1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/76218cb1 Branch: refs/heads/master Commit: 76218cb14260d068c1fb941646b72063d28fde7b Parents: 85b3d9e Author: Rajat Khandelwal <pro...@apache.org> Authored: Wed Mar 11 17:31:42 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Mar 11 17:31:42 2015 +0530 ---------------------------------------------------------------------- .../lens/cube/metadata/CubeMetastoreClient.java | 5 +- .../timeline/EndsAndHolesPartitionTimeline.java | 17 +++-- .../metadata/timeline/PartitionTimeline.java | 5 ++ .../timeline/StoreAllPartitionTimeline.java | 1 + .../timeline/TestPartitionTimelines.java | 74 ++++++++++++++++++++ 5 files changed, 97 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 2e03df1..8797bfe 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -172,6 +172,7 @@ public class CubeMetastoreClient { throws HiveException, LensException { String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage)); if (get(storageTableName) == null) { + log.info("loading timeline from all partitions for storage table: " + storageTableName); // not found in memory, try loading from table properties. Table storageTable = getTable(storageTableName); if (storageTable.getParameters().get(MetastoreUtil.getPartitoinTimelineCachePresenceKey()) == null) { @@ -256,7 +257,9 @@ public class CubeMetastoreClient { get(storageTable).get(updatePeriod).put(partitionColumn, PartitionTimelineFactory.get( CubeMetastoreClient.this, storageTable, updatePeriod, partitionColumn)); } - return get(storageTable).get(updatePeriod).get(partitionColumn); + PartitionTimeline ret = get(storageTable).get(updatePeriod).get(partitionColumn); + log.info("ensured entry " + ret); + return ret; } /** http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java index 57dae9e..d442bc0 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/EndsAndHolesPartitionTimeline.java @@ -43,7 +43,7 @@ import lombok.NonNull; @Data public class EndsAndHolesPartitionTimeline extends PartitionTimeline { private TimePartition first; - private TreeSet<TimePartition> holes = new TreeSet<TimePartition>(); + private TreeSet<TimePartition> holes = Sets.newTreeSet(); private TimePartition latest; public EndsAndHolesPartitionTimeline(CubeMetastoreClient client, String storageTableName, UpdatePeriod updatePeriod, @@ -122,10 +122,19 @@ public class EndsAndHolesPartitionTimeline extends PartitionTimeline { @Override public boolean initFromProperties(Map<String, String> properties) throws LensException { - first = TimePartition.of(getUpdatePeriod(), properties.get("first")); - latest = TimePartition.of(getUpdatePeriod(), properties.get("latest")); - holes = Sets.newTreeSet(); + first = null; + latest = null; + holes.clear(); + String firstStr = properties.get("first"); + String latestStr = properties.get("latest"); String holesStr = properties.get("holes"); + if (!Strings.isNullOrEmpty(firstStr)) { + first = TimePartition.of(getUpdatePeriod(), firstStr); + } + if (!Strings.isNullOrEmpty(latestStr)) { + latest = TimePartition.of(getUpdatePeriod(), latestStr); + } + holes = Sets.newTreeSet(); if (!Strings.isNullOrEmpty(holesStr)) { for (String hole : properties.get("holes").split("\\s*,\\s*")) { holes.add(TimePartition.of(getUpdatePeriod(), hole)); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java index 64771d0..30921df 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java @@ -31,6 +31,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.Data; import lombok.NonNull; +import lombok.extern.apachecommons.CommonsLog; /** * Represents the in-memory data structure that represents timeline of all existing partitions for a given storage @@ -40,6 +41,7 @@ import lombok.NonNull; * @see org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline */ @Data +@CommonsLog public abstract class PartitionTimeline implements Iterable<TimePartition> { private final CubeMetastoreClient client; private final String storageTableName; @@ -84,7 +86,9 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> { props.put(entry.getKey().substring(prefix.length()), entry.getValue()); } } + log.info("initializing timeline: " + getStorageTableName() + ", " + getUpdatePeriod() + ", " + getPartCol()); initFromProperties(props); + log.info("initialized to " + toProperties()); } /** @@ -111,6 +115,7 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> { } boolean result = add(getAll()); all = null; + log.info("after commit batch additions, timeline is: " + this); return result; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java index 8a9450c..c79c9ce 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/StoreAllPartitionTimeline.java @@ -78,6 +78,7 @@ public class StoreAllPartitionTimeline extends PartitionTimeline { @Override public boolean initFromProperties(Map<String, String> properties) throws LensException { + allPartitions.clear(); String partitionsStr = properties.get("partitions"); if (partitionsStr == null) { return true; http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/76218cb1/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java new file mode 100644 index 0000000..0027e64 --- /dev/null +++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.metadata.timeline; + +import java.util.Date; +import java.util.Map; + +import org.apache.lens.api.LensException; +import org.apache.lens.cube.metadata.CubeMetastoreClient; +import org.apache.lens.cube.metadata.TimePartition; +import org.apache.lens.cube.metadata.UpdatePeriod; + +import org.testng.Assert; +import org.testng.annotations.Test; + +public class TestPartitionTimelines { + CubeMetastoreClient client = null; + private static final String TABLE_NAME = "storage_fact"; + private static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY; + private static final String PART_COL = "pt"; + private static final Date DATE = new Date(); + + @Test + public void testPropertiesContractsForAllSubclasses() throws LensException { + testPropertiesContract(StoreAllPartitionTimeline.class); + testPropertiesContract(EndsAndHolesPartitionTimeline.class); + } + + private <T extends PartitionTimeline> T getInstance(Class<T> clz) { + try { + return clz.getConstructor(CubeMetastoreClient.class, String.class, UpdatePeriod.class, String.class) + .newInstance(client, TABLE_NAME, PERIOD, PART_COL); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + private <T extends PartitionTimeline> void testPropertiesContract(Class<T> clz) throws LensException { + T inst1 = getInstance(clz); + T inst2 = getInstance(clz); + Map<String, String> props = inst1.toProperties(); + Assert.assertTrue(inst2.initFromProperties(props)); + Assert.assertEquals(inst1, inst2); + Assert.assertTrue(inst1.isEmpty()); + Assert.assertTrue(inst2.isEmpty()); + Assert.assertTrue(inst1.add(TimePartition.of(PERIOD, DATE))); + Assert.assertFalse(inst1.equals(inst2)); + Assert.assertTrue(inst2.add(TimePartition.of(PERIOD, DATE))); + Assert.assertTrue(inst1.isConsistent()); + Assert.assertTrue(inst2.isConsistent()); + Assert.assertEquals(inst1, inst2); + Assert.assertTrue(inst2.initFromProperties(props)); + Assert.assertFalse(inst1.equals(inst2)); + Assert.assertTrue(inst2.initFromProperties(inst1.toProperties())); + Assert.assertEquals(inst1, inst2); + } +}
