This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 46d8603 [GOBBLIN-1149] Abstract out method for constructing
descriptor from config
46d8603 is described below
commit 46d8603bcdccd41bffdcb83e1175766fe57af8f8
Author: Jack Moseley <[email protected]>
AuthorDate: Tue May 19 08:29:15 2020 -0700
[GOBBLIN-1149] Abstract out method for constructing descriptor from config
Closes #2987 from jack-moseley/construct-
descriptor
---
.../modules/dataset/DatasetDescriptorUtils.java | 34 ++++++++++++++++++++++
.../flowgraph/pathfinder/AbstractPathFinder.java | 12 ++------
.../modules/template/StaticFlowTemplate.java | 12 ++------
3 files changed, 40 insertions(+), 18 deletions(-)
diff --git
a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/dataset/DatasetDescriptorUtils.java
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/dataset/DatasetDescriptorUtils.java
new file mode 100644
index 0000000..e21a31d
--- /dev/null
+++
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/dataset/DatasetDescriptorUtils.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.service.modules.dataset;
+
+import com.typesafe.config.Config;
+
+import
org.apache.gobblin.service.modules.flowgraph.DatasetDescriptorConfigKeys;
+import org.apache.gobblin.util.reflection.GobblinConstructorUtils;
+
+
+public class DatasetDescriptorUtils {
+ /**
+ * Given dataset descriptor config, construct a {@link DatasetDescriptor}
object
+ */
+ public static DatasetDescriptor constructDatasetDescriptor(Config
descriptorConfig) throws ReflectiveOperationException {
+ Class datasetDescriptorClass =
Class.forName(descriptorConfig.getString(DatasetDescriptorConfigKeys.CLASS_KEY));
+ return (DatasetDescriptor)
GobblinConstructorUtils.invokeLongestConstructor(datasetDescriptorClass,
descriptorConfig);
+ }
+}
diff --git
a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/flowgraph/pathfinder/AbstractPathFinder.java
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/flowgraph/pathfinder/AbstractPathFinder.java
index 741fdfd..7b2c5cb 100644
---
a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/flowgraph/pathfinder/AbstractPathFinder.java
+++
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/flowgraph/pathfinder/AbstractPathFinder.java
@@ -41,6 +41,7 @@ import org.apache.gobblin.runtime.api.SpecExecutor;
import org.apache.gobblin.runtime.api.SpecNotFoundException;
import org.apache.gobblin.service.ServiceConfigKeys;
import org.apache.gobblin.service.modules.dataset.DatasetDescriptor;
+import org.apache.gobblin.service.modules.dataset.DatasetDescriptorUtils;
import org.apache.gobblin.service.modules.flow.FlowEdgeContext;
import org.apache.gobblin.service.modules.flow.FlowGraphPath;
import org.apache.gobblin.service.modules.flow.FlowUtils;
@@ -150,15 +151,8 @@ public abstract class AbstractPathFinder implements
PathFinder {
destDatasetDescriptorConfig =
destDatasetDescriptorConfig.withFallback(getDefaultConfig(this.destNodes.get(0)));
}
- Class srcdatasetDescriptorClass =
-
Class.forName(srcDatasetDescriptorConfig.getString(DatasetDescriptorConfigKeys.CLASS_KEY));
- this.srcDatasetDescriptor = (DatasetDescriptor) GobblinConstructorUtils
- .invokeLongestConstructor(srcdatasetDescriptorClass,
srcDatasetDescriptorConfig);
- Class destDatasetDescriptorClass =
-
Class.forName(destDatasetDescriptorConfig.getString(DatasetDescriptorConfigKeys.CLASS_KEY));
- this.destDatasetDescriptor = (DatasetDescriptor) GobblinConstructorUtils
- .invokeLongestConstructor(destDatasetDescriptorClass,
destDatasetDescriptorConfig);
-
+ this.srcDatasetDescriptor =
DatasetDescriptorUtils.constructDatasetDescriptor(srcDatasetDescriptorConfig);
+ this.destDatasetDescriptor =
DatasetDescriptorUtils.constructDatasetDescriptor(destDatasetDescriptorConfig);
}
private Config getDefaultConfig(DataNode dataNode) {
diff --git
a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/template/StaticFlowTemplate.java
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/template/StaticFlowTemplate.java
index e833897..c13e032 100644
---
a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/template/StaticFlowTemplate.java
+++
b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/template/StaticFlowTemplate.java
@@ -44,9 +44,9 @@ import org.apache.gobblin.runtime.api.SpecNotFoundException;
import org.apache.gobblin.runtime.job_spec.JobSpecResolver;
import org.apache.gobblin.runtime.job_spec.ResolvedJobSpec;
import org.apache.gobblin.service.modules.dataset.DatasetDescriptor;
+import org.apache.gobblin.service.modules.dataset.DatasetDescriptorUtils;
import
org.apache.gobblin.service.modules.flowgraph.DatasetDescriptorConfigKeys;
import
org.apache.gobblin.service.modules.template_catalog.FlowCatalogWithTemplates;
-import org.apache.gobblin.util.reflection.GobblinConstructorUtils;
/**
@@ -122,11 +122,11 @@ public class StaticFlowTemplate implements FlowTemplate {
while (config.hasPath(inputPrefix)) {
try {
Config inputDescriptorConfig = config.getConfig(inputPrefix);
- DatasetDescriptor inputDescriptor =
getDatasetDescriptor(inputDescriptorConfig);
+ DatasetDescriptor inputDescriptor =
DatasetDescriptorUtils.constructDatasetDescriptor(inputDescriptorConfig);
String outputPrefix = Joiner.on(".")
.join(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX,
Integer.toString(i));
Config outputDescriptorConfig = config.getConfig(outputPrefix);
- DatasetDescriptor outputDescriptor =
getDatasetDescriptor(outputDescriptorConfig);
+ DatasetDescriptor outputDescriptor =
DatasetDescriptorUtils.constructDatasetDescriptor(outputDescriptorConfig);
if (resolvable) {
try {
@@ -146,12 +146,6 @@ public class StaticFlowTemplate implements FlowTemplate {
return result;
}
- private DatasetDescriptor getDatasetDescriptor(Config descriptorConfig)
- throws ReflectiveOperationException {
- Class datasetDescriptorClass =
Class.forName(descriptorConfig.getString(DatasetDescriptorConfigKeys.CLASS_KEY));
- return (DatasetDescriptor)
GobblinConstructorUtils.invokeLongestConstructor(datasetDescriptorClass,
descriptorConfig);
- }
-
@Override
public Config getRawTemplateConfig() {
return this.rawConfig;
