Repository: incubator-gobblin Updated Branches: refs/heads/master 841193a37 -> 4e9453fba
[GOBBLIN-56] Changing statestore checker to work with any state store Making non-required the state store url Adding required text to help for required properties Fixing issue where it read second paramater as value due to using hasArgs instead of hasArg Closes #2077 from treff7es/mysql_state_store_checker Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/4e9453fb Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/4e9453fb Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/4e9453fb Branch: refs/heads/master Commit: 4e9453fba6e1ceb51077ec1fe5871f4d65109ff1 Parents: 841193a Author: treff7es <[email protected]> Authored: Mon Aug 28 09:01:13 2017 -0700 Committer: Hung Tran <[email protected]> Committed: Mon Aug 28 09:01:13 2017 -0700 ---------------------------------------------------------------------- .../runtime/util/JobStateToJsonConverter.java | 34 +++++++++++------- .../util/JobStateToJsonConverterTest.java | 38 +++++++++++++++++--- 2 files changed, 55 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4e9453fb/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java index 5860daa..71a66db 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java @@ -33,9 +33,8 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,9 +42,10 @@ import com.google.common.io.Closer; import com.google.gson.stream.JsonWriter; import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.metastore.DatasetStateStore; import org.apache.gobblin.metastore.StateStore; -import org.apache.gobblin.runtime.FsDatasetStateStore; import org.apache.gobblin.runtime.JobState; +import org.apache.gobblin.util.ConfigUtils; import org.apache.gobblin.util.JobConfigurationUtils; @@ -66,11 +66,13 @@ public class JobStateToJsonConverter { public JobStateToJsonConverter(Properties props, String storeUrl, boolean keepConfig) throws IOException { Configuration conf = new Configuration(); JobConfigurationUtils.putPropertiesIntoConfiguration(props, conf); - Path storePath = new Path(storeUrl); - FileSystem fs = storePath.getFileSystem(conf); - String storeRootDir = storePath.toUri().getPath(); - this.jobStateStore = new FsDatasetStateStore(fs, storeRootDir); + + if (StringUtils.isNotBlank(storeUrl)) { + props.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, storeUrl); + } + this.keepConfig = keepConfig; + this.jobStateStore = (StateStore) DatasetStateStore.buildDatasetStateStore(ConfigUtils.propertiesToConfig(props)); } /** @@ -154,19 +156,19 @@ public class JobStateToJsonConverter { @SuppressWarnings("all") public static void main(String[] args) throws Exception { Option sysConfigOption = Option.builder("sc").argName("system configuration file") - .desc("Gobblin system configuration file").longOpt("sysconfig").hasArgs().build(); + .desc("Gobblin system configuration file (required if no state store URL specified)").longOpt("sysconfig").hasArg().build(); Option storeUrlOption = Option.builder("u").argName("gobblin state store URL") - .desc("Gobblin state store root path URL").longOpt("storeurl").hasArgs().required().build(); - Option jobNameOption = Option.builder("n").argName("gobblin job name").desc("Gobblin job name").longOpt("name") - .hasArgs().required().build(); + .desc("Gobblin state store root path URL (required if no sysconfig specified)").longOpt("storeurl").hasArg().build(); + Option jobNameOption = Option.builder("n").argName("gobblin job name").desc("Gobblin job name (required)").longOpt("name") + .hasArg().required().build(); Option jobIdOption = - Option.builder("i").argName("gobblin job id").desc("Gobblin job id").longOpt("id").hasArgs().build(); + Option.builder("i").argName("gobblin job id").desc("Gobblin job id").longOpt("id").hasArg().build(); Option convertAllOption = Option.builder("a").desc("Whether to convert all past job states of the given job").longOpt("all").build(); Option keepConfigOption = Option.builder("kc").desc("Whether to keep all configuration properties").longOpt("keepConfig").build(); Option outputToFile = - Option.builder("t").argName("output file name").desc("Output file name").longOpt("toFile").hasArgs().build(); + Option.builder("t").argName("output file name").desc("Output file name").longOpt("toFile").hasArg().build(); Options options = new Options(); options.addOption(sysConfigOption); @@ -187,6 +189,12 @@ public class JobStateToJsonConverter { System.exit(1); } + if (!cmd.hasOption(sysConfigOption.getLongOpt()) && !cmd.hasOption(storeUrlOption.getLongOpt()) ){ + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("JobStateToJsonConverter", options); + System.exit(1); + } + Properties sysConfig = new Properties(); if (cmd.hasOption(sysConfigOption.getLongOpt())) { sysConfig = JobConfigurationUtils.fileToProperties(cmd.getOptionValue(sysConfigOption.getLongOpt())); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4e9453fb/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java ---------------------------------------------------------------------- diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java index e4bf4f9..66ca54f 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/util/JobStateToJsonConverterTest.java @@ -16,17 +16,22 @@ */ package org.apache.gobblin.runtime.util; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.stream.JsonReader; import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; import java.util.Properties; + import org.testng.Assert; import org.testng.annotations.Test; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.stream.JsonReader; + +import org.apache.gobblin.configuration.ConfigurationKeys; + + @Test(groups = { "gobblin.runtime" }) public class JobStateToJsonConverterTest { @@ -40,7 +45,9 @@ public class JobStateToJsonConverterTest { public void testJsonKeepConfig() throws IOException { String stateStorePath = getClass().getClassLoader().getResource(TEST_STORE).getPath(); + boolean keepConfig = true; + JobStateToJsonConverter converter = new JobStateToJsonConverter(new Properties(), stateStorePath, keepConfig); StringWriter stringWriter = new StringWriter(); @@ -53,4 +60,27 @@ public class JobStateToJsonConverterTest { Assert.assertNotNull(taskState.getAsJsonObject().get(PROPERTIES)); } } + + @Test + public void testJsonKeepConfigWithoutStoreUrl() + throws IOException { + String stateStorePath = getClass().getClassLoader().getResource(TEST_STORE).getPath(); + Properties properties = new Properties(); + + properties.setProperty(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, stateStorePath); + + boolean keepConfig = true; + + JobStateToJsonConverter converter = new JobStateToJsonConverter(properties, null, keepConfig); + + StringWriter stringWriter = new StringWriter(); + converter.convert(TEST_JOB, stringWriter); + + JsonObject json = new JsonParser().parse(new JsonReader(new StringReader(stringWriter.toString()))).getAsJsonObject(); + + Assert.assertNotNull(json.get(PROPERTIES)); + for (JsonElement taskState: json.get(TASK_STATES).getAsJsonArray()) { + Assert.assertNotNull(taskState.getAsJsonObject().get(PROPERTIES)); + } + } }
