Repository: incubator-gobblin Updated Branches: refs/heads/master 695863e82 -> 4b19fedb2
[GOBBLIN-279] pull file unable to reuse the json property. GOBBLIN-279 Code intendation Adding missed test job pull file Code Indentation as per Gobblin Coding Standard Closes #2132 from dallaybatta/Gobblin-279 Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/4b19fedb Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/4b19fedb Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/4b19fedb Branch: refs/heads/master Commit: 4b19fedb24596b90ffde26aadb29080177649a75 Parents: 695863e Author: [email protected] <[email protected]> Authored: Tue Dec 12 01:30:54 2017 -0800 Committer: Abhishek Tiwari <[email protected]> Committed: Tue Dec 12 01:30:54 2017 -0800 ---------------------------------------------------------------------- .../org/apache/gobblin/util/PullFileLoader.java | 5 +++++ .../apache/gobblin/util/PullFileLoaderTest.java | 14 +++++++++++++- .../test/resources/pullFileLoaderTest/bjob.pull | 20 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4b19fedb/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java index 083fc75..a68f9ac 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/PullFileLoader.java @@ -72,6 +72,9 @@ public class PullFileLoader { public static final Set<String> DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS = Sets.newHashSet("pull", "job"); public static final Set<String> DEFAULT_HOCON_PULL_FILE_EXTENSIONS = Sets.newHashSet("json", "conf"); + public static final String PROPERTY_DELIMITER_PARSING_ENABLED_KEY = "property.parsing.enablekey"; + public static final boolean DEFAULT_PROPERTY_DELIMITER_PARSING_ENABLED_KEY = false; + private final Path rootDirectory; private final FileSystem fs; private final ExtensionFilter javaPropsPullFileFilter; @@ -260,6 +263,8 @@ public class PullFileLoader { PropertiesConfiguration propertiesConfiguration = new PropertiesConfiguration(); try (InputStreamReader inputStreamReader = new InputStreamReader(this.fs.open(propertiesPath), Charsets.UTF_8)) { + propertiesConfiguration.setDelimiterParsingDisabled(ConfigUtils.getBoolean(fallback, + PROPERTY_DELIMITER_PARSING_ENABLED_KEY, DEFAULT_PROPERTY_DELIMITER_PARSING_ENABLED_KEY)); propertiesConfiguration.load(inputStreamReader); Config configFromProps = http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4b19fedb/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java b/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java index 87261cd..2560206 100644 --- a/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java +++ b/gobblin-utility/src/test/java/org/apache/gobblin/util/PullFileLoaderTest.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path; import org.testng.Assert; import org.testng.annotations.Test; +import com.google.common.collect.ImmutableMap; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; @@ -218,7 +219,18 @@ public class PullFileLoaderTest { Assert.assertEquals(pullFile.entrySet().size(), 7); } - + @Test + public void testJsonPropertyReuseJobLoading() throws Exception { + Path path; + Config pullFile; + path = new Path(this.basePath, "bjob.pull"); + Config cfg = ConfigFactory.parseMap(ImmutableMap.<String, Object>builder() + .put(PullFileLoader.PROPERTY_DELIMITER_PARSING_ENABLED_KEY, true) + .build()); + pullFile = loader.loadPullFile(path, cfg, false); + Assert.assertEquals(pullFile.getString("json.property.key"), pullFile.getString("json.property.key1")); + } + private Config pullFileFromPath(Collection<Config> configs, Path path) throws IOException { for (Config config : configs) { if (config.getString(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY).equals(path.toString())) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4b19fedb/gobblin-utility/src/test/resources/pullFileLoaderTest/bjob.pull ---------------------------------------------------------------------- diff --git a/gobblin-utility/src/test/resources/pullFileLoaderTest/bjob.pull b/gobblin-utility/src/test/resources/pullFileLoaderTest/bjob.pull new file mode 100644 index 0000000..10bf811 --- /dev/null +++ b/gobblin-utility/src/test/resources/pullFileLoaderTest/bjob.pull @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +json.property.key={"name":"shankar","country":"bharat"} +json.property.key1=${json.property.key} +
