Updated Branches: refs/heads/sqoop2 3e4107f25 -> eab812d2f
SQOOP-1258. Sqoop2: FrameworkMetadataUpgrade can't handle new inputs (Jarek Jarcec Cecho via Hari Shreedharan) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/eab812d2 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/eab812d2 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/eab812d2 Branch: refs/heads/sqoop2 Commit: eab812d2f2f75416a8bd02599c18501a56609315 Parents: 3e4107f Author: Hari Shreedharan <[email protected]> Authored: Fri Dec 13 19:24:47 2013 -0800 Committer: Hari Shreedharan <[email protected]> Committed: Fri Dec 13 19:24:47 2013 -0800 ---------------------------------------------------------------------- .../GenericJdbcConnectorMetadataUpgrader.java | 1 - .../framework/FrameworkMetadataUpgrader.java | 20 ++- .../TestFrameworkMetadataUpgrader.java | 170 +++++++++++++++++++ 3 files changed, 188 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/eab812d2/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java index 864426c..2b12009 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorMetadataUpgrader.java @@ -25,7 +25,6 @@ import org.apache.sqoop.model.MConnectionForms; import org.apache.sqoop.model.MForm; import org.apache.sqoop.model.MInput; import org.apache.sqoop.model.MJobForms; -import org.apache.sqoop.validation.Status; import java.util.HashMap; import java.util.List; http://git-wip-us.apache.org/repos/asf/sqoop/blob/eab812d2/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java b/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java index ef00780..2437fa6 100644 --- a/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java +++ b/core/src/main/java/org/apache/sqoop/framework/FrameworkMetadataUpgrader.java @@ -18,6 +18,8 @@ */ package org.apache.sqoop.framework; +import org.apache.log4j.Logger; +import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.spi.MetadataUpgrader; import org.apache.sqoop.model.MConnectionForms; import org.apache.sqoop.model.MForm; @@ -29,6 +31,9 @@ import java.util.List; import java.util.Map; public class FrameworkMetadataUpgrader extends MetadataUpgrader{ + + private static final Logger LOG = Logger.getLogger(FrameworkMetadataUpgrader.class); + @Override public void upgrade(MConnectionForms original, MConnectionForms upgradeTarget) { @@ -54,9 +59,20 @@ public class FrameworkMetadataUpgrader extends MetadataUpgrader{ for (MForm form : target) { List<MInput<?>> inputs = form.getInputs(); MForm originalForm = formMap.get(form.getName()); + if(originalForm == null) { + LOG.warn("Form: " + form.getName() + " not present in old " + + "framework metadata. So it will not be transferred by the upgrader."); + continue; + } + for (MInput input : inputs) { - MInput originalInput = originalForm.getInput(input.getName()); - input.setValue(originalInput.getValue()); + try { + MInput originalInput = originalForm.getInput(input.getName()); + input.setValue(originalInput.getValue()); + } catch (SqoopException ex) { + LOG.warn("Input: " + input.getName() + " not present in old " + + "framework metadata. So it will not be transferred by the upgrader."); + } } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/eab812d2/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java new file mode 100644 index 0000000..cc0d984 --- /dev/null +++ b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkMetadataUpgrader.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sqoop.framework; + +import org.apache.sqoop.model.*; +import org.junit.Before; +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + */ +public class TestFrameworkMetadataUpgrader { + + FrameworkMetadataUpgrader upgrader; + + @Before + public void initializeUpgrader() { + upgrader = new FrameworkMetadataUpgrader(); + } + + /** + * We take the same forms on input and output and we + * expect that all values will be correctly transferred. + */ + @Test + public void testConnectionUpgrade() { + MConnectionForms original = connection1(); + MConnectionForms target = connection1(); + + original.getStringInput("f1.s1").setValue("A"); + original.getStringInput("f1.s2").setValue("B"); + original.getIntegerInput("f1.i").setValue(3); + + upgrader.upgrade(original, target); + + assertEquals("A", target.getStringInput("f1.s1").getValue()); + assertEquals("B", target.getStringInput("f1.s2").getValue()); + assertEquals(3, (long)target.getIntegerInput("f1.i").getValue()); + } + + /** + * We take the same forms on input and output and we + * expect that all values will be correctly transferred. + */ + @Test + public void testJobUpgrade() { + MJobForms original = job1(MJob.Type.IMPORT); + MJobForms target = job1(MJob.Type.IMPORT); + + original.getStringInput("f1.s1").setValue("A"); + original.getStringInput("f1.s2").setValue("B"); + original.getIntegerInput("f1.i").setValue(3); + + upgrader.upgrade(original, target); + + assertEquals("A", target.getStringInput("f1.s1").getValue()); + assertEquals("B", target.getStringInput("f1.s2").getValue()); + assertEquals(3, (long)target.getIntegerInput("f1.i").getValue()); + } + + /** + * Upgrade scenario when new input has been added to the target forms. + */ + @Test + public void testNonExistingInput() { + MConnectionForms original = connection1(); + MConnectionForms target = connection2(); + + original.getStringInput("f1.s1").setValue("A"); + original.getStringInput("f1.s2").setValue("B"); + original.getIntegerInput("f1.i").setValue(3); + + upgrader.upgrade(original, target); + + assertEquals("A", target.getStringInput("f1.s1").getValue()); + assertNull(target.getStringInput("f1.s2_").getValue()); + assertEquals(3, (long)target.getIntegerInput("f1.i").getValue()); + } + + /** + * Upgrade scenario when entire has been added in the target and + * therefore is missing in the original. + */ + @Test + public void testNonExistingForm() { + MConnectionForms original = connection1(); + MConnectionForms target = connection3(); + + original.getStringInput("f1.s1").setValue("A"); + original.getStringInput("f1.s2").setValue("B"); + original.getIntegerInput("f1.i").setValue(3); + + upgrader.upgrade(original, target); + + assertNull(target.getStringInput("f2.s1").getValue()); + assertNull(target.getStringInput("f2.s2").getValue()); + assertNull(target.getIntegerInput("f2.i").getValue()); + } + + MJobForms job1(MJob.Type type) { + return new MJobForms(type, forms1()); + } + + MConnectionForms connection1() { + return new MConnectionForms(forms1()); + } + + MConnectionForms connection2() { + return new MConnectionForms(forms2()); + } + + MConnectionForms connection3() { + return new MConnectionForms(forms3()); + } + + List<MForm> forms1() { + List<MForm> list = new LinkedList<MForm>(); + list.add(new MForm("f1", inputs1("f1"))); + return list; + } + + List<MInput<?>> inputs1(String formName) { + List<MInput<?>> list = new LinkedList<MInput<?>>(); + list.add(new MStringInput(formName + ".s1", false, (short)30)); + list.add(new MStringInput(formName + ".s2", false, (short)30)); + list.add(new MIntegerInput(formName + ".i", false)); + return list; + } + + List<MForm> forms2() { + List<MForm> list = new LinkedList<MForm>(); + list.add(new MForm("f1", inputs2("f1"))); + return list; + } + + List<MInput<?>> inputs2(String formName) { + List<MInput<?>> list = new LinkedList<MInput<?>>(); + list.add(new MStringInput(formName + ".s1", false, (short)30)); + list.add(new MStringInput(formName + ".s2_", false, (short)30)); + list.add(new MIntegerInput(formName + ".i", false)); + return list; + } + + List<MForm> forms3() { + List<MForm> list = new LinkedList<MForm>(); + list.add(new MForm("f2", inputs1("f2"))); + return list; + } +}
