Repository: incubator-gobblin Updated Branches: refs/heads/master 149ebd441 -> 47951b21d
[GOBBLIN-338] Adding setting to being able specify hive registration fs which is useful if you want to register data on other fs like s3. Closes #2194 from treff7es/configurable_hive_registration_filesystem Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/47951b21 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/47951b21 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/47951b21 Branch: refs/heads/master Commit: 47951b21d8e3379c4d731d05a109c49adcb7a925 Parents: 149ebd4 Author: treff7es <[email protected]> Authored: Tue Dec 12 01:22:24 2017 -0800 Committer: Abhishek Tiwari <[email protected]> Committed: Tue Dec 12 01:22:24 2017 -0800 ---------------------------------------------------------------------- .../gobblin/hive/avro/HiveAvroSerDeManager.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/47951b21/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java ---------------------------------------------------------------------- diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java index 96ebe83..b30a2fa 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java @@ -17,25 +17,28 @@ package org.apache.gobblin.hive.avro; -import com.codahale.metrics.Timer; -import org.apache.gobblin.instrumented.Instrumented; -import org.apache.gobblin.metrics.MetricContext; import java.io.IOException; +import java.net.URI; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import com.codahale.metrics.Timer; import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; + import org.apache.gobblin.annotation.Alpha; import org.apache.gobblin.configuration.State; import org.apache.gobblin.hive.HiveRegistrationUnit; import org.apache.gobblin.hive.HiveSerDeManager; import org.apache.gobblin.hive.HiveSerDeWrapper; +import org.apache.gobblin.hive.policy.HiveRegistrationPolicyBase; +import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.MetricContext; import org.apache.gobblin.util.AvroUtils; import org.apache.gobblin.util.HadoopUtils; -import lombok.extern.slf4j.Slf4j; /** @@ -68,7 +71,13 @@ public class HiveAvroSerDeManager extends HiveSerDeManager { public HiveAvroSerDeManager(State props) throws IOException { super(props); - this.fs = FileSystem.get(HadoopUtils.getConfFromState(props)); + + if (props.contains(HiveRegistrationPolicyBase.HIVE_FS_URI)) { + this.fs = FileSystem.get(URI.create(props.getProp(HiveRegistrationPolicyBase.HIVE_FS_URI)), HadoopUtils.getConfFromState(props)); + } else { + this.fs = FileSystem.get(HadoopUtils.getConfFromState(props)); + } + this.useSchemaFile = props.getPropAsBoolean(USE_SCHEMA_FILE, DEFAULT_USE_SCHEMA_FILE); this.schemaFileName = props.getProp(SCHEMA_FILE_NAME, DEFAULT_SCHEMA_FILE_NAME); this.schemaLiteralLengthLimit =
