Repository: kylin Updated Branches: refs/heads/master ab6e08351 -> acacb67db
KYLIN-2195 add BackwardCompatibilityConfig Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/acacb67d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/acacb67d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/acacb67d Branch: refs/heads/master Commit: acacb67db4aec8de1a8759579341d5d60fee04c7 Parents: ab6e083 Author: Yang Li <[email protected]> Authored: Tue Nov 15 22:14:52 2016 +0800 Committer: Yang Li <[email protected]> Committed: Wed Nov 16 08:32:52 2016 +0800 ---------------------------------------------------------------------- .../common/BackwardCompatibilityConfig.java | 109 +++++++++++++++++++ .../apache/kylin/common/KylinConfigBase.java | 9 +- .../org/apache/kylin/common/KylinConfigExt.java | 4 +- .../apache/kylin/common/KylinConfigTest.java | 38 ++++++- .../kylin-backward-compatibility.properties | 1 + .../test_case_data/localmeta/kylin.properties | 1 + 6 files changed, 155 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/acacb67d/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java new file mode 100644 index 0000000..0ca6f6d --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/BackwardCompatibilityConfig.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common; + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Maps; + +public class BackwardCompatibilityConfig { + + private static final Logger logger = LoggerFactory.getLogger(BackwardCompatibilityConfig.class); + static InputStream bccTestInput = null; + + private final Map<String, String> old2new = Maps.newConcurrentMap(); + + public BackwardCompatibilityConfig() { + InputStream is; + if (bccTestInput == null) { + // normal + is = Thread.currentThread().getContextClassLoader().getResourceAsStream("kylin-backward-compatibility.properties"); + } else { + // for test + is = bccTestInput; + bccTestInput = null; + } + if (is == null) + return; + + Properties props = new Properties(); + try { + props.load(is); + } catch (IOException e) { + logger.error("", e); + } finally { + IOUtils.closeQuietly(is); + } + + for (Entry<Object, Object> kv : props.entrySet()) { + old2new.put((String) kv.getKey(), (String) kv.getValue()); + } + } + + public String check(String key) { + String newKey = old2new.get(key); + if (newKey != null) { + logger.warn("Config '{}' is deprecated, use '{}' instead", key, newKey); + return newKey; + } else { + return key; + } + } + + public Map<String, String> check(Map<String, String> props) { + if (containsOldKey(props.keySet()) == false) + return props; + + LinkedHashMap<String, String> result = new LinkedHashMap<>(); + for (Entry<String, String> kv : props.entrySet()) { + result.put(check(kv.getKey()), kv.getValue()); + } + return result; + } + + public Properties check(Properties props) { + if (containsOldKey(props.keySet()) == false) + return props; + + Properties result = new Properties(); + for (Entry<Object, Object> kv : props.entrySet()) { + result.setProperty(check((String) kv.getKey()), (String) kv.getValue()); + } + return result; + } + + private boolean containsOldKey(Set<? extends Object> keySet) { + for (Object k : keySet) { + if (old2new.containsKey(k)) + return true; + } + return false; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/acacb67d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d44ff96..c116814 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -61,6 +61,9 @@ abstract public class KylinConfigBase implements Serializable { return kylinHome; } + // backward compatibility check happens when properties is loaded or updated + static final BackwardCompatibilityConfig CBC = new BackwardCompatibilityConfig(); + // ============================================================================ private volatile Properties properties = new Properties(); @@ -70,7 +73,7 @@ abstract public class KylinConfigBase implements Serializable { } public KylinConfigBase(Properties props) { - this.properties = props; + this.properties = CBC.check(props); } final protected String getOptional(String prop) { @@ -128,11 +131,11 @@ abstract public class KylinConfigBase implements Serializable { */ final public void setProperty(String key, String value) { logger.info("Kylin Config was updated with " + key + " : " + value); - properties.setProperty(key, value); + properties.setProperty(CBC.check(key), value); } final protected void reloadKylinConfig(Properties properties) { - this.properties = properties; + this.properties = CBC.check(properties); } // ============================================================================ http://git-wip-us.apache.org/repos/asf/kylin/blob/acacb67d/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java index c5eb300..7a4f4aa 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java @@ -44,13 +44,13 @@ public class KylinConfigExt extends KylinConfig { throw new IllegalArgumentException(); } this.base = base; - this.overrides = overrides; + this.overrides = CBC.check(overrides); } private KylinConfigExt(KylinConfigExt ext, Map<String, String> overrides) { super(ext.base.getAllProperties()); this.base = ext.base; - this.overrides = overrides; + this.overrides = CBC.check(overrides); } protected String getOptional(String prop, String dft) { http://git-wip-us.apache.org/repos/asf/kylin/blob/acacb67d/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java index 097fe11..99d0a43 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -18,17 +18,32 @@ package org.apache.kylin.common; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.util.Map; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import com.google.common.collect.Maps; + public class KylinConfigTest extends LocalFileMetadataTestCase { + @BeforeClass + static public void initBccTestInput() { + try { + BackwardCompatibilityConfig.bccTestInput = new FileInputStream(new File(LOCALMETA_TEST_DATA, "kylin-backward-compatibility.properties")); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } + @Before public void setUp() throws Exception { this.createTestMetadata(); @@ -47,5 +62,24 @@ public class KylinConfigTest extends LocalFileMetadataTestCase { assertEquals("test1", override.get("test1")); assertEquals("test2", override.get("test2")); } - + + @Test + public void testBackwardCompatibility() { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + final String oldk = "kylin.test.bcc.old.key"; + final String newk = "kylin.test.bcc.new.key"; + + assertNull(config.getOptional(oldk)); + assertNotNull(config.getOptional(newk)); + + Map<String, String> override = Maps.newHashMap(); + override.put(oldk, "1"); + KylinConfigExt ext = KylinConfigExt.createInstance(config, override); + assertEquals(ext.getOptional(oldk), null); + assertEquals(ext.getOptional(newk), "1"); + assertNotEquals(config.getOptional(newk), "1"); + + config.setProperty(oldk, "2"); + assertEquals(config.getOptional(newk), "2"); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/acacb67d/examples/test_case_data/localmeta/kylin-backward-compatibility.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin-backward-compatibility.properties b/examples/test_case_data/localmeta/kylin-backward-compatibility.properties new file mode 100644 index 0000000..67aef7a --- /dev/null +++ b/examples/test_case_data/localmeta/kylin-backward-compatibility.properties @@ -0,0 +1 @@ +kylin.test.bcc.old.key=kylin.test.bcc.new.key \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/acacb67d/examples/test_case_data/localmeta/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties index f1592d3..c46442e 100644 --- a/examples/test_case_data/localmeta/kylin.properties +++ b/examples/test_case_data/localmeta/kylin.properties @@ -127,6 +127,7 @@ mail.sender= ### OTHER ### # for tests +kylin.test.bcc.old.key=some-value kylin.job.mr.config.override.test1=test1 kylin.job.mr.config.override.test2=test2 kylin.job.controller.lock=org.apache.kylin.job.lock.MockJobLock
