Repository: incubator-gobblin Updated Branches: refs/heads/master 998fe200d -> d0ece1a04
[GOBBLIN-209] Support for hocon global configuration files. Closes #2061 from kadaan/add_support_for_hocon_global_files Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/d0ece1a0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/d0ece1a0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/d0ece1a0 Branch: refs/heads/master Commit: d0ece1a0436df88df2d305467d1d94c0893ad0bf Parents: 998fe20 Author: Joel Baranick <[email protected]> Authored: Tue Aug 15 08:22:34 2017 -0700 Committer: Issac Buenrostro <[email protected]> Committed: Tue Aug 15 08:22:34 2017 -0700 ---------------------------------------------------------------------- .../org/apache/gobblin/util/PullFileLoader.java | 29 ++++++++++++++++++-- .../apache/gobblin/util/PullFileLoaderTest.java | 12 +++++--- .../pullFileLoaderTest/dir1/dir1.configuration | 19 +++++++++++++ .../pullFileLoaderTest/dir1/dir1.properties | 18 ------------ 4 files changed, 54 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d0ece1a0/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java index 45884c9..083fc75 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java @@ -63,6 +63,12 @@ public class PullFileLoader { public static final String GLOBAL_PROPS_EXTENSION = ".properties"; public static final PathFilter GLOBAL_PROPS_PATH_FILTER = new ExtensionFilter(GLOBAL_PROPS_EXTENSION); + public static final String GLOBAL_HOCON_EXTENSION = ".configuration"; + public static final PathFilter GLOBAL_HOCON_PATH_FILTER = new ExtensionFilter(GLOBAL_HOCON_EXTENSION); + + public static final PathFilter GLOBAL_PATH_FILTER = + new ExtensionFilter(Lists.newArrayList(GLOBAL_PROPS_EXTENSION, GLOBAL_HOCON_EXTENSION)); + public static final Set<String> DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS = Sets.newHashSet("pull", "job"); public static final Set<String> DEFAULT_HOCON_PULL_FILE_EXTENSIONS = Sets.newHashSet("json", "conf"); @@ -224,7 +230,7 @@ public class PullFileLoader { * @throws IOException */ private Config findAndLoadGlobalConfigInDirectory(Path path, Config fallback) throws IOException { - FileStatus[] files = this.fs.listStatus(path, GLOBAL_PROPS_PATH_FILTER); + FileStatus[] files = this.fs.listStatus(path, GLOBAL_PATH_FILTER); if (files == null) { log.warn("Could not list files at path " + path); return ConfigFactory.empty(); @@ -232,7 +238,16 @@ public class PullFileLoader { if (files.length > 1) { throw new IOException("Found more than one global properties file at path " + path); } - return files.length == 1 ? loadJavaPropsWithFallback(files[0].getPath(), fallback) : fallback; + if (files.length == 0) { + return fallback; + } + if (GLOBAL_HOCON_PATH_FILTER.accept(files[0].getPath())) { + return loadHoconConfigWithFallback(files[0].getPath(), fallback); + } else if (GLOBAL_PROPS_PATH_FILTER.accept(files[0].getPath())) { + return loadJavaPropsWithFallback(files[0].getPath(), fallback); + } else { + throw new IllegalStateException("Unsupported global configuration file: " + files[0].getPath()); + } } /** @@ -268,4 +283,14 @@ public class PullFileLoader { } } + private Config loadHoconConfigWithFallback(Path path, Config fallback) throws IOException { + try (InputStream is = fs.open(path); + Reader reader = new InputStreamReader(is, Charsets.UTF_8)) { + return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, + PathUtils.getPathWithoutSchemeAndAuthority(path).toString())) + .withFallback(ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF))) + .withFallback(fallback); + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d0ece1a0/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java b/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java index ab1a72a..87261cd 100644 --- a/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java +++ b/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java @@ -159,20 +159,22 @@ public class PullFileLoaderTest { pullFile = loader.loadPullFile(path, ConfigUtils.propertiesToConfig(sysProps), true); Assert.assertEquals(pullFile.getString("key1"), "jobValue1,jobValue2,jobValue3"); Assert.assertEquals(pullFile.getString("key2"), "jobValue2"); + Assert.assertEquals(pullFile.getString("key2a"), "jobValue2"); Assert.assertEquals(pullFile.getString("key3"), "rootValue3"); Assert.assertEquals(pullFile.getString("key4"), "dir1Value4"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); - Assert.assertEquals(pullFile.entrySet().size(), 5); + Assert.assertEquals(pullFile.entrySet().size(), 6); path = new Path(this.basePath, "dir1/job.conf"); pullFile = loader.loadPullFile(path, ConfigFactory.empty(), true); Assert.assertEquals(pullFile.getString("key1"), "jobValue1,jobValue2,jobValue3"); Assert.assertEquals(pullFile.getString("key2"), "dir1Value4"); + Assert.assertEquals(pullFile.getString("key2a"), "dir1Value4"); Assert.assertEquals(pullFile.getString("key3"), "rootValue3"); Assert.assertEquals(pullFile.getString("key4"), "dir1Value4"); Assert.assertEquals(pullFile.getString("key10"), "jobValue2"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); - Assert.assertEquals(pullFile.entrySet().size(), 6); + Assert.assertEquals(pullFile.entrySet().size(), 7); } @Test @@ -198,20 +200,22 @@ public class PullFileLoaderTest { pullFile = pullFileFromPath(configs, path); Assert.assertEquals(pullFile.getString("key1"), "jobValue1,jobValue2,jobValue3"); Assert.assertEquals(pullFile.getString("key2"), "jobValue2"); + Assert.assertEquals(pullFile.getString("key2a"), "jobValue2"); Assert.assertEquals(pullFile.getString("key3"), "rootValue3"); Assert.assertEquals(pullFile.getString("key4"), "dir1Value4"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); - Assert.assertEquals(pullFile.entrySet().size(), 5); + Assert.assertEquals(pullFile.entrySet().size(), 6); path = new Path(this.basePath, "dir1/job.conf"); pullFile = pullFileFromPath(configs, path); Assert.assertEquals(pullFile.getString("key1"), "jobValue1,jobValue2,jobValue3"); Assert.assertEquals(pullFile.getString("key2"), "dir1Value4"); + Assert.assertEquals(pullFile.getString("key2a"), "dir1Value4"); Assert.assertEquals(pullFile.getString("key3"), "rootValue3"); Assert.assertEquals(pullFile.getString("key4"), "dir1Value4"); Assert.assertEquals(pullFile.getString("key10"), "jobValue2"); Assert.assertEquals(pullFile.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY), path.toString()); - Assert.assertEquals(pullFile.entrySet().size(), 6); + Assert.assertEquals(pullFile.entrySet().size(), 7); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d0ece1a0/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.configuration ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.configuration b/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.configuration new file mode 100644 index 0000000..8068d72 --- /dev/null +++ b/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.configuration @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +key4=dir1Value4 +key2a=${key2} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d0ece1a0/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.properties ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.properties b/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.properties deleted file mode 100644 index 023fa57..0000000 --- a/gobblin-utility/src/test/resources/pullFileLoaderTest/dir1/dir1.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -key4=dir1Value4
