Repository: incubator-gobblin Updated Branches: refs/heads/master a81a3288d -> 92f4255bf
[GOBBLIN-663] Fix Typesafe config resolution failure for non-HOCON strings Closes #2534 from sv2000/parseString Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/92f4255b Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/92f4255b Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/92f4255b Branch: refs/heads/master Commit: 92f4255bf93f698642076dd885c158baaf8c38de Parents: a81a328 Author: suvasude <[email protected]> Authored: Mon Jan 14 09:24:49 2019 -0800 Committer: Hung Tran <[email protected]> Committed: Mon Jan 14 09:24:49 2019 -0800 ---------------------------------------------------------------------- .../service/FlowConfigResourceLocalHandler.java | 16 +++- .../FlowConfigResourceLocalHandlerTest.java | 83 ++++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/92f4255b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/main/java/org/apache/gobblin/service/FlowConfigResourceLocalHandler.java ---------------------------------------------------------------------- diff --git a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/main/java/org/apache/gobblin/service/FlowConfigResourceLocalHandler.java b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/main/java/org/apache/gobblin/service/FlowConfigResourceLocalHandler.java index 580ab95..8ad0999 100644 --- a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/main/java/org/apache/gobblin/service/FlowConfigResourceLocalHandler.java +++ b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/main/java/org/apache/gobblin/service/FlowConfigResourceLocalHandler.java @@ -205,9 +205,19 @@ public class FlowConfigResourceLocalHandler implements FlowConfigsResourceHandle } Config config = configBuilder.build(); - //We allow for flowConfig to be in HOCON format. We first convert the StringMap object to a String object and - // then use ConfigFactory#parseString() to parse the HOCON string. - Config configWithFallback = config.withFallback(ConfigFactory.parseString(flowConfig.getProperties().toString()).resolve()); + + Config configWithFallback; + //We first attempt to process the REST.li request as a HOCON string. If the request is not a valid HOCON string + // (e.g. when certain special characters such as ":" or "*" are not properly escaped), we catch the Typesafe ConfigException and + // fallback to assuming that values are literal strings. + try { + // We first convert the StringMap object to a String object and then use ConfigFactory#parseString() to parse the + // HOCON string. + configWithFallback = config.withFallback(ConfigFactory.parseString(flowConfig.getProperties().toString()).resolve()); + } catch (Exception e) { + configWithFallback = config.withFallback(ConfigFactory.parseMap(flowConfig.getProperties())); + } + try { URI templateURI = new URI(flowConfig.getTemplateUris()); return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build(); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/92f4255b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/test/java/org/apache/gobblin/service/FlowConfigResourceLocalHandlerTest.java ---------------------------------------------------------------------- diff --git a/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/test/java/org/apache/gobblin/service/FlowConfigResourceLocalHandlerTest.java b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/test/java/org/apache/gobblin/service/FlowConfigResourceLocalHandlerTest.java new file mode 100644 index 0000000..4f4784f --- /dev/null +++ b/gobblin-restli/gobblin-flow-config-service/gobblin-flow-config-service-server/src/test/java/org/apache/gobblin/service/FlowConfigResourceLocalHandlerTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.google.common.collect.Maps; +import com.linkedin.data.template.StringMap; + +import org.apache.gobblin.runtime.api.FlowSpec; + + +public class FlowConfigResourceLocalHandlerTest { + private static final String FLOW_GROUP_KEY = "flow.group"; + private static final String FLOW_NAME_KEY = "flow.name"; + private static final String SCHEDULE_KEY = "job.schedule"; + private static final String RUN_IMMEDIATELY_KEY = "flow.runImmediately"; + + private static final String TEST_GROUP_NAME = "testGroup1"; + private static final String TEST_FLOW_NAME = "testFlow1"; + private static final String TEST_SCHEDULE = "0 1/0 * ? * *"; + private static final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; + + @Test + public void testCreateFlowSpecForConfig() throws URISyntaxException { + Map<String, String> flowProperties = Maps.newHashMap(); + flowProperties.put("param1", "a:b:c*.d"); + + FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) + .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). + setRunImmediately(true)) + .setProperties(new StringMap(flowProperties)); + + FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); + Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); + Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); + Assert.assertEquals(flowSpec.getConfig().getString(SCHEDULE_KEY), TEST_SCHEDULE); + Assert.assertEquals(flowSpec.getConfig().getBoolean(RUN_IMMEDIATELY_KEY), true); + Assert.assertEquals(flowSpec.getConfig().getString("param1"), "a:b:c*.d"); + Assert.assertEquals(flowSpec.getTemplateURIs().get().size(), 1); + Assert.assertTrue(flowSpec.getTemplateURIs().get().contains(new URI(TEST_TEMPLATE_URI))); + + flowProperties = Maps.newHashMap(); + flowProperties.put("param1", "value1"); + flowProperties.put("param2", "${param1}-123"); + flowProperties.put("param3", "\"a:b:c*.d\""); + flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) + .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). + setRunImmediately(true)) + .setProperties(new StringMap(flowProperties)); + flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); + + Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); + Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); + Assert.assertEquals(flowSpec.getConfig().getString(SCHEDULE_KEY), TEST_SCHEDULE); + Assert.assertEquals(flowSpec.getConfig().getBoolean(RUN_IMMEDIATELY_KEY), true); + Assert.assertEquals(flowSpec.getConfig().getString("param1"),"value1"); + Assert.assertEquals(flowSpec.getConfig().getString("param2"),"value1-123"); + Assert.assertEquals(flowSpec.getConfig().getString("param3"), "a:b:c*.d"); + Assert.assertEquals(flowSpec.getTemplateURIs().get().size(), 1); + Assert.assertTrue(flowSpec.getTemplateURIs().get().contains(new URI(TEST_TEMPLATE_URI))); + } +} \ No newline at end of file
