http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java index b689854..5d48a29 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java @@ -17,13 +17,13 @@ */ package org.apache.sqoop.connector.hdfs.configuration; -import org.apache.sqoop.model.FormClass; +import org.apache.sqoop.model.ConfigClass; import org.apache.sqoop.model.Input; -@FormClass +@ConfigClass public class LinkConfig { //Todo: Didn't find anything that belongs here... - // Since empty forms don't work (DERBYREPO_0008:The form contains no input metadata), I'm putting a dummy form here + // Since empty forms don't work (DERBYREPO_0008:The config contains no input metadata), I'm putting a dummy config here @Input(size = 255) public String dummy; }
http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java index 4970821..c0cd336 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfiguration.java @@ -18,14 +18,14 @@ package org.apache.sqoop.connector.hdfs.configuration; import org.apache.sqoop.model.ConfigurationClass; -import org.apache.sqoop.model.Form; +import org.apache.sqoop.model.Config; @ConfigurationClass public class LinkConfiguration { - @Form - public LinkConfig link; + @Config + public LinkConfig linkConfig; public LinkConfiguration() { - link = new LinkConfig(); + linkConfig = new LinkConfig(); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java index b1308db..2dfd738 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfig.java @@ -17,13 +17,13 @@ */ package org.apache.sqoop.connector.hdfs.configuration; -import org.apache.sqoop.model.FormClass; +import org.apache.sqoop.model.ConfigClass; import org.apache.sqoop.model.Input; /** * */ -@FormClass +@ConfigClass public class ToJobConfig { @Input public ToFormat outputFormat; http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java index bba249c..c91a975 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/ToJobConfiguration.java @@ -18,11 +18,11 @@ package org.apache.sqoop.connector.hdfs.configuration; import org.apache.sqoop.model.ConfigurationClass; -import org.apache.sqoop.model.Form; +import org.apache.sqoop.model.Config; @ConfigurationClass public class ToJobConfiguration { - @Form + @Config public ToJobConfig toJobConfig; public ToJobConfiguration() { http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties index b603f2f..9b8c6ba 100644 --- a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties +++ b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties @@ -18,12 +18,12 @@ ############################ # Link Config # -link.label = Link configuration -link.help = You must supply the information requested in order to \ +linkConfig.label = Link configuration +linkConfig.help = You must supply the information requested in order to \ create a connection object. -link.dummy.label = Dummy parameter needed to get HDFS connector to register -link.dummy.help = You can write anything here. Doesn't matter. +linkConfig.dummy.label = Dummy parameter needed to get HDFS connector to register +linkConfig.dummy.help = You can write anything here. Doesn't matter. # To Job Config # http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java index 765bedd..f5fbab7 100644 --- a/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/test/java/org/apache/sqoop/connector/idf/TestCSVIntermediateDataFormat.java @@ -18,6 +18,13 @@ */ package org.apache.sqoop.connector.idf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.UnsupportedEncodingException; +import java.util.Arrays; + import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.schema.Schema; import org.apache.sqoop.schema.type.Binary; @@ -26,13 +33,6 @@ import org.apache.sqoop.schema.type.Text; import org.junit.Before; import org.junit.Test; -import java.io.UnsupportedEncodingException; -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - public class TestCSVIntermediateDataFormat { private final String BYTE_FIELD_ENCODING = "ISO-8859-1"; http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java index dbfdc03..54bdd13 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java @@ -19,19 +19,18 @@ package org.apache.sqoop.connector; import java.io.IOException; import java.net.URL; -import java.util.ArrayList; import java.util.Properties; import org.apache.log4j.Logger; import org.apache.sqoop.common.Direction; -import org.apache.sqoop.core.ConfigurationConstants; -import org.apache.sqoop.model.FormUtils; -import org.apache.sqoop.model.MConnectionForms; -import org.apache.sqoop.model.MConnector; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.spi.SqoopConnector; -import org.apache.sqoop.model.MForm; -import org.apache.sqoop.model.MJobForms; +import org.apache.sqoop.core.ConfigurationConstants; +import org.apache.sqoop.model.ConfigUtils; +import org.apache.sqoop.model.MConnector; +import org.apache.sqoop.model.MFromConfig; +import org.apache.sqoop.model.MLinkConfig; +import org.apache.sqoop.model.MToConfig; public final class ConnectorHandler { @@ -92,26 +91,25 @@ public final class ConnectorHandler { connectorClassName, ex); } - // Initialize Metadata - MJobForms fromJobForms = null; - MJobForms toJobForms = null; + MFromConfig fromConfig = null; + MToConfig toConfig = null; if (connector.getSupportedDirections().contains(Direction.FROM)) { - fromJobForms = new MJobForms(FormUtils.toForms( + fromConfig = new MFromConfig(ConfigUtils.toConfigs( connector.getJobConfigurationClass(Direction.FROM))); } if (connector.getSupportedDirections().contains(Direction.TO)) { - toJobForms = new MJobForms(FormUtils.toForms( + toConfig = new MToConfig(ConfigUtils.toConfigs( connector.getJobConfigurationClass(Direction.TO))); } - MConnectionForms connectionForms = new MConnectionForms( - FormUtils.toForms(connector.getLinkConfigurationClass())); + MLinkConfig connectionForms = new MLinkConfig( + ConfigUtils.toConfigs(connector.getLinkConfigurationClass())); String connectorVersion = connector.getVersion(); mConnector = new MConnector(connectorUniqueName, connectorClassName, connectorVersion, - connectionForms, fromJobForms, toJobForms); + connectionForms, fromConfig, toConfig); if (LOG.isInfoEnabled()) { LOG.info("Connector [" + connectorClassName + "] initialized."); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java index c87df84..5226926 100644 --- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java +++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java @@ -119,7 +119,7 @@ public class ConnectorManager implements Reconfigurable { return handler.getConnector().getBundle(locale); } - public MConnector getConnectorMetadata(long connectorId) { + public MConnector getConnectorConfig(long connectorId) { ConnectorHandler handler = handlerMap.get(nameMap.get(connectorId)); if(handler == null) { return null; http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/Driver.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/Driver.java b/core/src/main/java/org/apache/sqoop/driver/Driver.java index 5297bde..f1b45bb 100644 --- a/core/src/main/java/org/apache/sqoop/driver/Driver.java +++ b/core/src/main/java/org/apache/sqoop/driver/Driver.java @@ -17,6 +17,7 @@ */ package org.apache.sqoop.driver; +import java.util.List; import java.util.Locale; import java.util.ResourceBundle; @@ -26,12 +27,12 @@ import org.apache.sqoop.core.ConfigurationConstants; import org.apache.sqoop.core.Reconfigurable; import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener; -import org.apache.sqoop.driver.configuration.JobConfiguration; -import org.apache.sqoop.driver.configuration.LinkConfiguration; -import org.apache.sqoop.model.FormUtils; -import org.apache.sqoop.model.MConnectionForms; +import org.apache.sqoop.driver.configuration.DriverConfiguration; +import org.apache.sqoop.json.DriverBean; +import org.apache.sqoop.model.ConfigUtils; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MJobForms; import org.apache.sqoop.repository.RepositoryManager; import org.apache.sqoop.validation.Validator; @@ -92,14 +93,14 @@ public class Driver implements Reconfigurable { } /** - * Driver config structure + * Driver structure */ - private MDriverConfig mDriverConfig; + private MDriver mDriver; /** * Validator instance */ - private final Validator validator; + private final Validator driverValidator; /** * Driver config upgrader instance @@ -111,38 +112,30 @@ public class Driver implements Reconfigurable { */ private static final boolean DEFAULT_AUTO_UPGRADE = false; - public static final String CURRENT_DRIVER_VERSION = "1"; - - public Class getJobConfigurationClass() { - return JobConfiguration.class; - } - - public Class getLinkConfigurationClass() { - return LinkConfiguration.class; + public Class getDriverConfigurationGroupClass() { + return DriverConfiguration.class; } public Driver() { - MConnectionForms connectionForms = new MConnectionForms( - FormUtils.toForms(getLinkConfigurationClass()) - ); - mDriverConfig = new MDriverConfig(connectionForms, new MJobForms(FormUtils.toForms(getJobConfigurationClass())), - CURRENT_DRIVER_VERSION); + List<MConfig> driverConfig = ConfigUtils.toConfigs(getDriverConfigurationGroupClass()); + mDriver = new MDriver(new MDriverConfig(driverConfig), DriverBean.CURRENT_DRIVER_VERSION); // Build validator - validator = new DriverValidator(); + driverValidator = new DriverConfigValidator(); // Build upgrader driverConfigUpgrader = new DriverConfigUpgrader(); } public synchronized void initialize() { - initialize(SqoopConfiguration.getInstance().getContext().getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE)); + initialize(SqoopConfiguration.getInstance().getContext() + .getBoolean(ConfigurationConstants.DRIVER_AUTO_UPGRADE, DEFAULT_AUTO_UPGRADE)); } public synchronized void initialize(boolean autoUpgrade) { LOG.trace("Begin Driver Config initialization"); // Register driver config in repository - mDriverConfig = RepositoryManager.getInstance().getRepository().registerDriverConfig(mDriverConfig, autoUpgrade); + mDriver = RepositoryManager.getInstance().getRepository().registerDriver(mDriver, autoUpgrade); SqoopConfiguration.getInstance().getProvider().registerListener(new CoreConfigurationListener(this)); @@ -154,15 +147,15 @@ public class Driver implements Reconfigurable { } public Validator getValidator() { - return validator; + return driverValidator; } public RepositoryUpgrader getDriverConfigRepositoryUpgrader() { return driverConfigUpgrader; } - public MDriverConfig getDriverConfig() { - return mDriverConfig; + public MDriver getDriver() { + return mDriver; } public ResourceBundle getBundle(Locale locale) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java index 8d6eb78..847b73d 100644 --- a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java +++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java @@ -18,56 +18,54 @@ */ package org.apache.sqoop.driver; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.spi.RepositoryUpgrader; -import org.apache.sqoop.model.MConnectionForms; -import org.apache.sqoop.model.MForm; +import org.apache.sqoop.model.MConfigList; +import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MInput; -import org.apache.sqoop.model.MJobForms; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import org.apache.sqoop.model.MLinkConfig; public class DriverConfigUpgrader extends RepositoryUpgrader{ private static final Logger LOG = Logger.getLogger(DriverConfigUpgrader.class); @Override - public void upgrade(MConnectionForms original, - MConnectionForms upgradeTarget) { - doUpgrade(original.getForms(), upgradeTarget.getForms()); + public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) { + // NOTE(VB): There are no link configs anymore for driver, this code remains for previous versions } @Override - public void upgrade(MJobForms original, MJobForms upgradeTarget) { - doUpgrade(original.getForms(), upgradeTarget.getForms()); - + public void upgrade(MConfigList original, MConfigList upgradeTarget) { + doUpgrade(original.getConfigs(), upgradeTarget.getConfigs()); } @SuppressWarnings("unchecked") - private void doUpgrade(List<MForm> original, List<MForm> target) { - // Easier to find the form in the original forms list if we use a map. - // Since the constructor of MJobForms takes a list, + private void doUpgrade(List<MConfig> original, List<MConfig> target) { + // Easier to find the config in the original list if we use a map. + // Since the constructor takes a list, // index is not guaranteed to be the same, so we need to look for // equivalence - Map<String, MForm> formMap = new HashMap<String, MForm>(); - for (MForm form : original) { - formMap.put(form.getName(), form); + Map<String, MConfig> configMap = new HashMap<String, MConfig>(); + for (MConfig config : original) { + configMap.put(config.getName(), config); } - for (MForm form : target) { - List<MInput<?>> inputs = form.getInputs(); - MForm originalForm = formMap.get(form.getName()); - if(originalForm == null) { - LOG.warn("Form: " + form.getName() + " not present in old " + + for (MConfig config : target) { + List<MInput<?>> inputs = config.getInputs(); + MConfig originalConfig = configMap.get(config.getName()); + if(originalConfig == null) { + LOG.warn("Config: " + config.getName() + " not present in old " + "driver config. So it will not be transferred by the upgrader."); continue; } for (MInput input : inputs) { try { - MInput originalInput = originalForm.getInput(input.getName()); + MInput originalInput = originalConfig.getInput(input.getName()); input.setValue(originalInput.getValue()); } catch (SqoopException ex) { LOG.warn("Input: " + input.getName() + " not present in old " + http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java new file mode 100644 index 0000000..9c3b660 --- /dev/null +++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.driver; + +import org.apache.sqoop.driver.configuration.DriverConfiguration; +import org.apache.sqoop.driver.configuration.ThrottlingConfig; +import org.apache.sqoop.validation.Status; +import org.apache.sqoop.validation.ConfigValidator; +import org.apache.sqoop.validation.Validator; + +public class DriverConfigValidator extends Validator { + @Override + public ConfigValidator validateConfigForJob(Object jobConfiguration) { + ConfigValidator validation = new ConfigValidator(DriverConfiguration.class); + DriverConfiguration conf = (DriverConfiguration)jobConfiguration; + validateThrottlingConfig(validation,conf.throttlingConfig); + + return validation; + }; + + private void validateThrottlingConfig(ConfigValidator validation, ThrottlingConfig throttlingConfig) { + if(throttlingConfig.numExtractors != null && throttlingConfig.numExtractors < 1) { + validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numExtractors", "You need to specify more than one extractor"); + } + + if(throttlingConfig.numLoaders != null && throttlingConfig.numLoaders < 1) { + validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numLoaders", "You need to specify more than one loader"); + } + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java deleted file mode 100644 index 9cc51dd..0000000 --- a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.driver; - -import org.apache.sqoop.driver.configuration.LinkConfiguration; -import org.apache.sqoop.driver.configuration.JobConfiguration; -import org.apache.sqoop.driver.configuration.ThrottlingForm; -import org.apache.sqoop.validation.Status; -import org.apache.sqoop.validation.Validation; -import org.apache.sqoop.validation.Validator; - -public class DriverValidator extends Validator { - @Override - public Validation validateLink(Object linkConfiguration) { - Validation validation = new Validation(LinkConfiguration.class); - // No validation on link object - return validation; - } - - @Override - public Validation validateJob(Object jobConfiguration) { - Validation validation = new Validation(JobConfiguration.class); - JobConfiguration conf = (JobConfiguration)jobConfiguration; - validateThrottlingForm(validation,conf.throttling); - - return validation; - }; - - private void validateThrottlingForm(Validation validation, ThrottlingForm throttling) { - if(throttling.extractors != null && throttling.extractors < 1) { - validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor"); - } - - if(throttling.loaders != null && throttling.loaders < 1) { - validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader"); - } - } - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/JobManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java index e91c436..df2a5ab 100644 --- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java +++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java @@ -30,13 +30,13 @@ import org.apache.sqoop.connector.spi.SqoopConnector; import org.apache.sqoop.core.Reconfigurable; import org.apache.sqoop.core.SqoopConfiguration; import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener; -import org.apache.sqoop.driver.configuration.JobConfiguration; +import org.apache.sqoop.driver.configuration.DriverConfiguration; import org.apache.sqoop.job.etl.Destroyer; import org.apache.sqoop.job.etl.DestroyerContext; import org.apache.sqoop.job.etl.Initializer; import org.apache.sqoop.job.etl.InitializerContext; import org.apache.sqoop.job.etl.Transferable; -import org.apache.sqoop.model.FormUtils; +import org.apache.sqoop.model.ConfigUtils; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MSubmission; @@ -311,35 +311,27 @@ public class JobManager implements Reconfigurable { SqoopConnector toConnector = getConnector(toConnection.getConnectorId()); validateSupportedDirection(toConnector, Direction.TO); - // Transform config to fromConnector specific classes - Object fromConnectionConfig = ClassUtils.instantiate(fromConnector - .getLinkConfigurationClass()); - FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig); + // link config for the FROM part of the job + Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass()); + ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig); - // Transform config to toConnector specific classes - Object toConnectorConfig = ClassUtils - .instantiate(toConnector.getLinkConfigurationClass()); - FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig); + // link config for the TO part of the job + Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass()); + ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig); + // from config for the job Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM)); - FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob); + ConfigUtils.fromConfigs(job.getJobConfig(Direction.FROM).getConfigs(), fromJob); + // to config for the job Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO)); - FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob); + ConfigUtils.fromConfigs(job.getJobConfig(Direction.TO).getConfigs(), toJob); - // Transform framework specific configs - // Q(VB) : Aren't the following 2 exactly the same? - Object fromDriverConnection = ClassUtils.instantiate(Driver.getInstance() - .getLinkConfigurationClass()); - FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromDriverConnection); + // the only driver config for the job + Object driverConfig = ClassUtils + .instantiate(Driver.getInstance().getDriverConfigurationGroupClass()); + ConfigUtils.fromConfigs(job.getDriverConfig().getConfigs(), driverConfig); - Object toDriverConnection = ClassUtils.instantiate(Driver.getInstance() - .getLinkConfigurationClass()); - FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toDriverConnection); - - Object frameworkJob = ClassUtils.instantiate(Driver.getInstance() - .getJobConfigurationClass()); - FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob); // Create a job request for submit/execution JobRequest jobRequest = executionEngine.createJobRequest(); @@ -347,14 +339,14 @@ public class JobManager implements Reconfigurable { jobRequest.setSummary(submission); jobRequest.setConnector(Direction.FROM, fromConnector); jobRequest.setConnector(Direction.TO, toConnector); - jobRequest.setConnectorLinkConfig(Direction.FROM, fromConnectionConfig); - jobRequest.setConnectorLinkConfig(Direction.TO, toConnectorConfig); - jobRequest.setConnectorJobConfig(Direction.FROM, fromJob); - jobRequest.setConnectorJobConfig(Direction.TO, toJob); - // TODO(Abe): Should we actually have 2 different Driver Connection config objects? - jobRequest.setFrameworkLinkConfig(Direction.FROM, fromDriverConnection); - jobRequest.setFrameworkLinkConfig(Direction.TO, toDriverConnection); - jobRequest.setFrameworkJobConfig(frameworkJob); + + jobRequest.setConnectorLinkConfig(Direction.FROM, fromLinkConfig); + jobRequest.setConnectorLinkConfig(Direction.TO, toLinkConfig); + + jobRequest.setJobConfig(Direction.FROM, fromJob); + jobRequest.setJobConfig(Direction.TO, toJob); + + jobRequest.setDriverConfig(driverConfig); jobRequest.setJobName(job.getName()); jobRequest.setJobId(job.getPersistenceId()); jobRequest.setNotificationUrl(notificationBaseUrl + jobId); @@ -453,12 +445,12 @@ public class JobManager implements Reconfigurable { // Initialize submission from the connector perspective initializer.initialize(initializerContext, jobRequest.getConnectorLinkConfig(direction), - jobRequest.getConnectorJobConfig(direction)); + jobRequest.getJobConfig(direction)); return initializer.getSchema(initializerContext, jobRequest.getConnectorLinkConfig(direction), - jobRequest.getConnectorJobConfig(direction)); + jobRequest.getJobConfig(direction)); } private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) { @@ -468,7 +460,7 @@ public class JobManager implements Reconfigurable { // Add job specific jars to jobRequest.addJars(initializer.getJars(initializerContext, jobRequest.getConnectorLinkConfig(direction), - jobRequest.getConnectorJobConfig(direction))); + jobRequest.getJobConfig(direction))); } private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) { @@ -488,12 +480,12 @@ public class JobManager implements Reconfigurable { } void prepareJob(JobRequest request) { - JobConfiguration jobConfiguration = (JobConfiguration) request.getFrameworkJobConfig(); + DriverConfiguration jobConfiguration = (DriverConfiguration) request.getDriverConfig(); // We're directly moving configured number of extractors and loaders to // underlying request object. In the future we might need to throttle this // count based on other running jobs to meet our SLAs. - request.setExtractors(jobConfiguration.throttling.extractors); - request.setLoaders(jobConfiguration.throttling.loaders); + request.setExtractors(jobConfiguration.throttlingConfig.numExtractors); + request.setLoaders(jobConfiguration.throttlingConfig.numLoaders); // Delegate rest of the job to execution engine executionEngine.prepareJob(request); @@ -532,9 +524,9 @@ public class JobManager implements Reconfigurable { // destroy submission from connector perspective fromDestroyer.destroy(fromDestroyerContext, request.getConnectorLinkConfig(Direction.FROM), - request.getConnectorJobConfig(Direction.FROM)); + request.getJobConfig(Direction.FROM)); toDestroyer.destroy(toDestroyerContext, request.getConnectorLinkConfig(Direction.TO), - request.getConnectorJobConfig(Direction.TO)); + request.getJobConfig(Direction.TO)); } public MSubmission stop(long jobId, HttpEventContext ctx) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/JobRequest.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java index 63e1e49..2666320 100644 --- a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java +++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java @@ -78,11 +78,11 @@ public class JobRequest { */ Object fromConnectorLinkConfig; Object toConnectorLinkConfig; - Object fromConnectorJobConfig; - Object toConnectorJobConfig; - Object fromFrameworkLinkConfig; - Object toFrameworkLinkConfig; - Object frameworkJobConfig; + + Object fromConfig; + Object toConfig; + + Object driverConfig; /** * Connector context (submission specific configuration) @@ -124,10 +124,9 @@ public class JobRequest { this.toConnector = null; this.fromConnectorLinkConfig = null; this.toConnectorLinkConfig = null; - this.fromConnectorJobConfig = null; - this.toConnectorJobConfig = null; - this.fromFrameworkLinkConfig = null; - this.toFrameworkLinkConfig = null; + this.fromConfig = null; + this.toConfig = null; + this.driverConfig = null; } public MSubmission getSummary() { @@ -244,64 +243,38 @@ public class JobRequest { } } - public Object getConnectorJobConfig(Direction type) { - switch(type) { - case FROM: - return fromConnectorJobConfig; - - case TO: - return toConnectorJobConfig; - - default: - throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type); - } - } - - public void setConnectorJobConfig(Direction type, Object config) { - switch(type) { - case FROM: - fromConnectorJobConfig = config; - break; - case TO: - toConnectorJobConfig = config; - break; - default: - throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type); - } - } - - public Object getFrameworkLinkConfig(Direction type) { + public Object getJobConfig(Direction type) { switch(type) { case FROM: - return fromFrameworkLinkConfig; + return fromConfig; case TO: - return toFrameworkLinkConfig; + return toConfig; default: throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type); } } - public void setFrameworkLinkConfig(Direction type, Object config) { + public void setJobConfig(Direction type, Object config) { switch(type) { case FROM: - fromFrameworkLinkConfig = config; + fromConfig = config; break; case TO: - toFrameworkLinkConfig = config; + toConfig = config; break; default: throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type); } } - public Object getFrameworkJobConfig() { - return frameworkJobConfig; + public Object getDriverConfig() { + return driverConfig; } - public void setFrameworkJobConfig(Object config) { - frameworkJobConfig = config; + public void setDriverConfig(Object config) { + driverConfig = config; } public MutableMapContext getConnectorContext(Direction type) { http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java new file mode 100644 index 0000000..d4e2254 --- /dev/null +++ b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.driver.configuration; + +import org.apache.sqoop.model.ConfigurationClass; +import org.apache.sqoop.model.Config; + +/** + * Representing the core job configuration + */ +@ConfigurationClass +public class DriverConfiguration { + @Config + public ThrottlingConfig throttlingConfig; + + public DriverConfiguration() { + throttlingConfig = new ThrottlingConfig(); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java deleted file mode 100644 index 908a4eb..0000000 --- a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.driver.configuration; - -import org.apache.sqoop.model.ConfigurationClass; -import org.apache.sqoop.model.Form; - -/** - * Representing the core job configuration - */ -@ConfigurationClass -public class JobConfiguration { - @Form - public ThrottlingForm throttling; - - public JobConfiguration() { - throttling = new ThrottlingForm(); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java deleted file mode 100644 index 3202844..0000000 --- a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.driver.configuration; - -import org.apache.sqoop.model.ConfigurationClass; - -/** - * Representing the core link configuration - */ -@ConfigurationClass -public class LinkConfiguration { - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java new file mode 100644 index 0000000..357d8e5 --- /dev/null +++ b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.driver.configuration; + +import org.apache.sqoop.model.ConfigClass; +import org.apache.sqoop.model.Input; + +/** + * Config to set up number of loaders and extractors + */ +@ConfigClass +public class ThrottlingConfig { + + @Input public Integer numExtractors; + + @Input public Integer numLoaders; +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java deleted file mode 100644 index e73007e..0000000 --- a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.driver.configuration; - -import org.apache.sqoop.model.FormClass; -import org.apache.sqoop.model.Input; - -/** - * Form to set up number of loaders and extractors - */ -@FormClass -public class ThrottlingForm { - - @Input public Integer extractors; - - @Input public Integer loaders; -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java index 3466116..3ade247 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java @@ -25,7 +25,7 @@ import org.apache.log4j.Logger; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MConnector; -import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MSubmission; @@ -51,7 +51,7 @@ public class JdbcRepository extends Repository { /** * Do what is needed to be done with given link object. * - * @param conn Connection to metadata repository. + * @param conn Connection to the repository. * @return Arbitrary value */ Object doIt(Connection conn) throws Exception; @@ -158,8 +158,8 @@ public class JdbcRepository extends Repository { public Object doIt(Connection conn) throws Exception { String connectorUniqueName = mConnector.getUniqueName(); - MConnector result = handler.findConnector(connectorUniqueName, conn); - if (result == null) { + MConnector connectorResult = handler.findConnector(connectorUniqueName, conn); + if (connectorResult == null) { handler.registerConnector(mConnector, conn); return mConnector; } else { @@ -167,23 +167,23 @@ public class JdbcRepository extends Repository { // For now, use the "string" versions itself - later we should // probably include a build number or something that is // monotonically increasing. - if (result.getUniqueName().equals(mConnector.getUniqueName()) && - mConnector.getVersion().compareTo(result.getVersion()) > 0) { + if (connectorResult.getUniqueName().equals(mConnector.getUniqueName()) && + mConnector.getVersion().compareTo(connectorResult.getVersion()) > 0) { if (autoUpgrade) { - upgradeConnector(result, mConnector); + upgradeConnector(connectorResult, mConnector); return mConnector; } else { throw new SqoopException(RepositoryError.JDBCREPO_0026, "Connector: " + mConnector.getUniqueName()); } } - if (!result.equals(mConnector)) { + if (!connectorResult.equals(mConnector)) { throw new SqoopException(RepositoryError.JDBCREPO_0013, "Connector: " + mConnector.getUniqueName() + " given: " + mConnector - + " found: " + result); + + " found: " + connectorResult); } - return result; + return connectorResult; } } }); @@ -220,27 +220,27 @@ public class JdbcRepository extends Repository { * {@inheritDoc} */ @Override - public MDriverConfig registerDriverConfig(final MDriverConfig mDriverConfig, final boolean autoUpgrade) { - return (MDriverConfig) doWithConnection(new DoWithConnection() { + public MDriver registerDriver(final MDriver mDriver, final boolean autoUpgrade) { + return (MDriver) doWithConnection(new DoWithConnection() { @Override public Object doIt(Connection conn) { - MDriverConfig result = handler.findDriverConfig(conn); - if (result == null) { - handler.registerDriverConfig(mDriverConfig, conn); - return mDriverConfig; + MDriver existingDriverConfig = handler.findDriver(conn); + if (existingDriverConfig == null) { + handler.registerDriver(mDriver, conn); + return mDriver; } else { // We're currently not serializing version into repository // so let's just compare the structure to see if we need upgrade. - if(!mDriverConfig.equals(result)) { + if(!mDriver.equals(existingDriverConfig)) { if (autoUpgrade) { - upgradeDriverConfig(mDriverConfig); - return mDriverConfig; + upgradeDriver(mDriver); + return mDriver; } else { throw new SqoopException(RepositoryError.JDBCREPO_0026, - "DriverConfig: " + mDriverConfig.getPersistenceId()); + "DriverConfig: " + mDriver.getPersistenceId()); } } - return result; + return existingDriverConfig; } } }); @@ -664,11 +664,11 @@ public class JdbcRepository extends Repository { } - protected void updateDriverConfig(final MDriverConfig mDriverConfig, RepositoryTransaction tx) { + protected void updateDriver(final MDriver mDriver, RepositoryTransaction tx) { doWithConnection(new DoWithConnection() { @Override public Object doIt(Connection conn) throws Exception { - handler.updateDriverConfig(mDriverConfig, conn); + handler.updateDriver(mDriver, conn); return null; } }, (JdbcRepositoryTransaction) tx); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java index a743491..97de893 100644 --- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java +++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.sqoop.model.MLink; import org.apache.sqoop.model.MConnector; -import org.apache.sqoop.model.MDriverConfig; +import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MJob; import org.apache.sqoop.model.MSubmission; @@ -113,11 +113,11 @@ public abstract class JdbcRepositoryHandler { * which may not have changed). After this operation the repository is * guaranteed to only have the new forms specified in this object. * - * @param mDriverConfig The new data to be inserted into the repository for + * @param mDriver The new data to be inserted into the repository for * the driverConfig. * @param conn JDBC link for querying repository */ - public abstract void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn); + public abstract void updateDriver(MDriver mDriver, Connection conn); /** @@ -127,7 +127,7 @@ public abstract class JdbcRepositoryHandler { * @return null if driverConfig are not yet present in repository or * loaded representation. */ - public abstract MDriverConfig findDriverConfig(Connection conn); + public abstract MDriver findDriver(Connection conn); /** * Register driver config in repository. @@ -138,7 +138,7 @@ public abstract class JdbcRepositoryHandler { * @param driverConfig Driver config that should be registered. * @param conn JDBC link for querying repository. */ - public abstract void registerDriverConfig(MDriverConfig driverConfig, Connection conn); + public abstract void registerDriver(MDriver driverConfig, Connection conn); /** * Return true if repository tables exists and are suitable for use. @@ -243,7 +243,7 @@ public abstract class JdbcRepositoryHandler { * * @param linkId Link id * @param conn Connection to the repository - * @return Deserialized form of the link that is saved in repository + * @return Deserialized config of the link that is saved in repository */ public abstract MLink findLink(long linkId, Connection conn); @@ -323,7 +323,7 @@ public abstract class JdbcRepositoryHandler { * * @param jobId Job id * @param conn Connection to the repository - * @return Deserialized form of the job that is present in the repository + * @return Deserialized config of the job that is present in the repository */ public abstract MJob findJob(long jobId, Connection conn); http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/Repository.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java index ea1608a..95c7a4d 100644 --- a/core/src/main/java/org/apache/sqoop/repository/Repository.java +++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java @@ -28,18 +28,21 @@ import org.apache.sqoop.connector.ConnectorManager; import org.apache.sqoop.connector.spi.RepositoryUpgrader; import org.apache.sqoop.connector.spi.SqoopConnector; import org.apache.sqoop.driver.Driver; -import org.apache.sqoop.model.FormUtils; -import org.apache.sqoop.model.MConnectionForms; +import org.apache.sqoop.json.DriverBean; +import org.apache.sqoop.model.ConfigUtils; +import org.apache.sqoop.model.MConfig; import org.apache.sqoop.model.MConnector; +import org.apache.sqoop.model.MDriver; import org.apache.sqoop.model.MDriverConfig; -import org.apache.sqoop.model.MForm; +import org.apache.sqoop.model.MFromConfig; import org.apache.sqoop.model.MJob; -import org.apache.sqoop.model.MJobForms; import org.apache.sqoop.model.MLink; +import org.apache.sqoop.model.MLinkConfig; import org.apache.sqoop.model.MPersistableEntity; import org.apache.sqoop.model.MSubmission; +import org.apache.sqoop.model.MToConfig; import org.apache.sqoop.utils.ClassUtils; -import org.apache.sqoop.validation.Validation; +import org.apache.sqoop.validation.ConfigValidator; import org.apache.sqoop.validation.Validator; @@ -76,15 +79,26 @@ public abstract class Repository { /** * Registers given connector in the repository and return registered - * variant. This method might return an exception in case that - * given connector are already registered with different structure. + * variant. This method might return an exception in case that + * given connector are already registered with different structure * * @param mConnector the connector to be registered - * autoupgrade whether to upgrade driver config automatically + * @param autoUpgrade whether to upgrade driver config automatically * @return Registered connector structure */ public abstract MConnector registerConnector(MConnector mConnector, boolean autoUpgrade); + /** + * Registers given driver and its config in the repository and return registered + * variant. This method might return an exception in case that the + * given driverConfig are already registered with different structure + * + * @param mDriverConfig driverConfig to be registered + * @param autoUpgrade whether to upgrade driverConfig automatically + * @return Registered connector structure + */ + public abstract MDriver registerDriver(MDriver mDriverConfig, boolean autoUpgrade); + /** * Search for connector with given name in repository. * @@ -103,18 +117,6 @@ public abstract class Repository { */ public abstract List<MConnector> findConnectors(); - - /** - * Registers given driverConfig in the repository and return registered - * variant. This method might return an exception in case that the - * given driverConfig are already registered with different structure. - * - * @param mDriverConfig driverConfig to be registered - * autoupgrade whether to upgrade driverConfig automatically - * @return Registered connector structure - */ - public abstract MDriverConfig registerDriverConfig(MDriverConfig mDriverConfig, boolean autoUpgrade); - /** * Save given link to repository. This link must not be already * present in the repository otherwise exception will be thrown. @@ -164,7 +166,7 @@ public abstract class Repository { * Find link with given id in repository. * * @param id Link id - * @return Deserialized form of the link that is saved in repository + * @return Deserialized config of the link that is saved in repository */ public abstract MLink findLink(long id); @@ -184,7 +186,7 @@ public abstract class Repository { public abstract void createJob(MJob job); /** - * Update given job metadata in repository. This object must already be saved + * Update given job entity in repository. This object must already be saved * in repository otherwise exception will be thrown. * * @param job Job object that should be updated in the repository @@ -192,7 +194,7 @@ public abstract class Repository { public abstract void updateJob(MJob job); /** - * Update given job metadata in repository. This object must already be saved + * Update given job entity in repository. This object must already be saved * in repository otherwise exception will be thrown. * * @param job Job object that should be updated in the repository @@ -204,7 +206,7 @@ public abstract class Repository { public abstract void updateJob(MJob job, RepositoryTransaction tx); /** - * Enable or disable job with given id from metadata repository + * Enable or disable job with given id from entity repository * * @param id Job object that is going to be enabled or disabled * @param enabled Enable or disable @@ -212,7 +214,7 @@ public abstract class Repository { public abstract void enableJob(long id, boolean enabled); /** - * Delete job with given id from metadata repository. + * Delete job with given id from entity repository. * * @param id Job id that should be removed */ @@ -222,7 +224,7 @@ public abstract class Repository { * Find job object with given id. * * @param id Job id - * @return Deserialized form of job loaded from repository + * @return Deserialized config of job loaded from repository */ public abstract MJob findJob(long id); @@ -288,8 +290,7 @@ public abstract class Repository { * @param connectorID Connector ID whose links should be fetched * @return List of MLink that use <code>connectorID</code>. */ - public abstract List<MLink> findLinksForConnector(long - connectorID); + public abstract List<MLink> findLinksForConnector(long connectorID); /** * Retrieve jobs which use the given link. @@ -297,17 +298,16 @@ public abstract class Repository { * @param connectorID Connector ID whose jobs should be fetched * @return List of MJobs that use <code>linkID</code>. */ - public abstract List<MJob> findJobsForConnector(long - connectorID); + public abstract List<MJob> findJobsForConnector(long connectorID); /** * Update the connector with the new data supplied in the - * <tt>newConnector</tt>. Also Update all forms associated with this - * connector in the repository with the forms specified in + * <tt>newConnector</tt>. Also Update all configs associated with this + * connector in the repository with the configs specified in * <tt>mConnector</tt>. <tt>mConnector </tt> must - * minimally have the connectorID and all required forms (including ones + * minimally have the connectorID and all required configs (including ones * which may not have changed). After this operation the repository is - * guaranteed to only have the new forms specified in this object. + * guaranteed to only have the new configs specified in this object. * * @param newConnector The new data to be inserted into the repository for * this connector. @@ -319,22 +319,22 @@ public abstract class Repository { protected abstract void updateConnector(MConnector newConnector, RepositoryTransaction tx); /** - * Update the driverConfig with the new data supplied in the - * <tt>mDriverConfig</tt>. Also Update all forms associated with the driverConfig - * in the repository with the forms specified in + * Update the driver with the new data supplied in the + * <tt>mDriverConfig</tt>. Also Update all configs associated with the driverConfig + * in the repository with the configs specified in * <tt>mDriverConfig</tt>. <tt>mDriverConfig </tt> must - * minimally have the connectorID and all required forms (including ones + * minimally have the connectorID and all required configs (including ones * which may not have changed). After this operation the repository is - * guaranteed to only have the new forms specified in this object. + * guaranteed to only have the new configs specified in this object. * - * @param mDriverConfig The new data to be inserted into the repository for + * @param mDriver The new data to be inserted into the repository for * the driverConfig. * @param tx The repository transaction to use to push the data to the * repository. If this is null, a new transaction will be created. * method will not call begin, commit, * rollback or close on this transaction. */ - protected abstract void updateDriverConfig(MDriverConfig mDriverConfig, RepositoryTransaction tx); + protected abstract void updateDriver(MDriver mDriver, RepositoryTransaction tx); /** * Delete all inputs for a job @@ -365,6 +365,13 @@ public abstract class Repository { } } + private void deleteJobs(List<MJob> jobs, RepositoryTransaction tx) { + for (MJob job : jobs) { + deleteJobInputs(job.getPersistenceId(), tx); + } + } + + /** * Upgrade the connector with the same {@linkplain MConnector#uniqueName} * in the repository with values from <code>newConnector</code>. @@ -377,7 +384,7 @@ public abstract class Repository { * upgraded. */ public final void upgradeConnector(MConnector oldConnector, MConnector newConnector) { - LOG.info("Upgrading metadata for connector: " + oldConnector.getUniqueName()); + LOG.info("Upgrading connector: " + oldConnector.getUniqueName()); long connectorID = oldConnector.getPersistenceId(); newConnector.setPersistenceId(connectorID); /* Algorithms: @@ -385,8 +392,8 @@ public abstract class Repository { * 2. Get all links associated with the connector. * 3. Get all jobs associated with the connector. * 4. Delete the inputs for all of the jobs and links (in that order) - * 5. Remove all inputs and forms associated with the connector, and - * register the new forms and inputs. + * 5. Remove all inputs and configs associated with the connector, and + * register the new configs and inputs. * 6. Create new links and jobs with connector part being the ones * returned by the upgrader. * 7. Validate new links and jobs with connector's validator @@ -401,85 +408,63 @@ public abstract class Repository { ConnectorManager.getInstance().getConnector(newConnector .getUniqueName()); - Validator validator = connector.getValidator(); - + Validator connectorConfigValidator = connector.getConfigValidator(); boolean upgradeSuccessful = true; - RepositoryUpgrader upgrader = connector.getRepositoryUpgrader(); - List<MLink> links = findLinksForConnector( - connectorID); - List<MJob> jobs = findJobsForConnector(connectorID); + List<MLink> linksByConnector = findLinksForConnector(connectorID); + List<MJob> jobsByConnector = findJobsForConnector(connectorID); // -- BEGIN TXN -- tx = getTransaction(); tx.begin(); - deletelinksAndJobs(links, jobs, tx); + deletelinksAndJobs(linksByConnector, jobsByConnector, tx); updateConnector(newConnector, tx); - for (MLink link : links) { - // Make a new copy of the forms from the connector, - // else the values will get set in the forms in the connector for - // each link. - List<MForm> forms = newConnector.getConnectionForms().clone(false).getForms(); - MConnectionForms newlinkForms = new MConnectionForms(forms); - upgrader.upgrade(link.getConnectorPart(), newlinkForms); - MLink newlink = new MLink(link, newlinkForms, link.getFrameworkPart()); - - // Transform form structures to objects for validations + for (MLink oldLink : linksByConnector) { + // Make a new copy of the configs + List<MConfig> linkConfig = newConnector.getLinkConfig().clone(false).getConfigs(); + MLinkConfig newLinkConfig = new MLinkConfig(linkConfig); + MLinkConfig oldLinkConfig = oldLink.getConnectorLinkConfig(); + upgrader.upgrade(oldLinkConfig, newLinkConfig); + + MLink newlink = new MLink(oldLink, newLinkConfig); + Object newConfigurationObject = ClassUtils.instantiate(connector.getLinkConfigurationClass()); - FormUtils.fromForms(newlink.getConnectorPart().getForms(), newConfigurationObject); + ConfigUtils.fromConfigs(newlink.getConnectorLinkConfig().getConfigs(), newConfigurationObject); - Validation validation = validator.validateLink(newConfigurationObject); - if (validation.getStatus().canProceed()) { + ConfigValidator configValidator = connectorConfigValidator.validateConfigForLink(newConfigurationObject); + if (configValidator.getStatus().canProceed()) { updateLink(newlink, tx); } else { - logInvalidModelObject("link", newlink, validation); + logInvalidModelObject("link", newlink, configValidator); upgradeSuccessful = false; } } - for (MJob job : jobs) { - // Make a new copy of the forms from the connector, - // else the values will get set in the forms in the connector for + for (MJob job : jobsByConnector) { + // Make a new copy of the configs + // else the values will get set in the configs in the connector for // each job. - List<MForm> fromForms = newConnector.getJobForms(Direction.FROM).clone(false).getForms(); - List<MForm> toForms = newConnector.getJobForms(Direction.TO).clone(false).getForms(); + List<MConfig> fromConfig = newConnector.getConfig(Direction.FROM).clone(false).getConfigs(); + List<MConfig> toConfig = newConnector.getConfig(Direction.TO).clone(false).getConfigs(); - // New FROM direction forms, old TO direction forms. + // New FROM direction configs, old TO direction configs. if (job.getConnectorId(Direction.FROM) == newConnector.getPersistenceId()) { - MJobForms newFromJobForms = new MJobForms(fromForms); - MJobForms oldToJobForms = job.getConnectorPart(Direction.TO); - upgrader.upgrade(job.getConnectorPart(Direction.FROM), newFromJobForms); - MJob newJob = new MJob(job, newFromJobForms, oldToJobForms, job.getFrameworkPart()); - updateJob(newJob, tx); + MFromConfig newFromConfig = new MFromConfig(fromConfig); + MFromConfig oldFromCOnfig = job.getFromJobConfig(); + upgrader.upgrade(oldFromCOnfig, newFromConfig); - // Transform form structures to objects for validations -// Object newFromConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.FROM)); -// FormUtils.fromForms(newJob.getConnectorPart(Direction.FROM).getForms(), newFromConfigurationObject); -// Validation fromValidation = validator.validateJob(newFromConfigurationObject); -// if (fromValidation.getStatus().canProceed()) { -// updateJob(newJob, tx); -// } else { -// logInvalidModelObject("job", newJob, fromValidation); -// upgradeSuccessful = false; -// } + MToConfig oldToConfig = job.getToJobConfig(); + MJob newJob = new MJob(job, newFromConfig, oldToConfig, job.getDriverConfig()); + updateJob(newJob, tx); } - // Old FROM direction forms, new TO direction forms. + // Old FROM direction configs, new TO direction configs. if (job.getConnectorId(Direction.TO) == newConnector.getPersistenceId()) { - MJobForms oldFromJobForms = job.getConnectorPart(Direction.FROM); - MJobForms newToJobForms = new MJobForms(toForms); - upgrader.upgrade(job.getConnectorPart(Direction.TO), newToJobForms); - MJob newJob = new MJob(job, oldFromJobForms, newToJobForms, job.getFrameworkPart()); - updateJob(newJob, tx); - // Transform form structures to objects for validations -// Object newToConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.TO)); -// FormUtils.fromForms(newJob.getConnectorPart(Direction.TO).getForms(), newToConfigurationObject); -// Validation toValidation = validator.validateJob(newToConfigurationObject); -// if (toValidation.getStatus().canProceed()) { -// updateJob(newJob, tx); -// } else { -// logInvalidModelObject("job", newJob, toValidation); -// upgradeSuccessful = false; -// } + MToConfig oldToConfig = job.getToJobConfig(); + MToConfig newToConfig = new MToConfig(toConfig); + upgrader.upgrade(oldToConfig, newToConfig); + MFromConfig oldFromConfig = job.getFromJobConfig(); + MJob newJob = new MJob(job, oldFromConfig, newToConfig, job.getDriverConfig()); + updateJob(newJob, tx); } } @@ -506,60 +491,35 @@ public abstract class Repository { } } - public final void upgradeDriverConfig(MDriverConfig driverConfig) { - LOG.info("Upgrading driver config"); + public final void upgradeDriver(MDriver driver) { + LOG.info("Upgrading driver"); RepositoryTransaction tx = null; try { - RepositoryUpgrader upgrader = Driver.getInstance() + RepositoryUpgrader driverConfigUpgrader = Driver.getInstance() .getDriverConfigRepositoryUpgrader(); - List<MLink> links = findLinks(); List<MJob> jobs = findJobs(); Validator validator = Driver.getInstance().getValidator(); - boolean upgradeSuccessful = true; // -- BEGIN TXN -- tx = getTransaction(); tx.begin(); - deletelinksAndJobs(links, jobs, tx); - updateDriverConfig(driverConfig, tx); - for (MLink link : links) { - // Make a new copy of the forms from the connector, - // else the values will get set in the forms in the connector for - // each link. - // @TODO(Abe): From/To link forms. - List<MForm> forms = driverConfig.getConnectionForms().clone(false).getForms(); - MConnectionForms newlinkForms = new MConnectionForms(forms); - upgrader.upgrade(link.getFrameworkPart(), newlinkForms); - MLink newlink = new MLink(link, link.getConnectorPart(), newlinkForms); - - // Transform form structures to objects for validations - Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass()); - FormUtils.fromForms(newlink.getFrameworkPart().getForms(), newConfigurationObject); - - Validation validation = validator.validateLink(newConfigurationObject); - if (validation.getStatus().canProceed()) { - updateLink(newlink, tx); - } else { - logInvalidModelObject("link", newlink, validation); - upgradeSuccessful = false; - } - } + deleteJobs(jobs, tx); + updateDriver(driver, tx); + for (MJob job : jobs) { - // Make a new copy of the forms from the framework, - // else the values will get set in the forms in the connector for - // each link. - List<MForm> forms = driverConfig.getJobForms().clone(false).getForms(); - MJobForms newJobForms = new MJobForms(forms); - upgrader.upgrade(job.getFrameworkPart(), newJobForms); - MJob newJob = new MJob(job, job.getConnectorPart(Direction.FROM), job.getConnectorPart(Direction.TO), newJobForms); - - // Transform form structures to objects for validations - Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass()); - FormUtils.fromForms(newJob.getFrameworkPart().getForms(), newConfigurationObject); - - Validation validation = validator.validateJob(newConfigurationObject); + // Make a new copy of the configs + MDriverConfig driverConfig = driver.getDriverConfig().clone(false); + MDriver newDriver = new MDriver(driverConfig, DriverBean.CURRENT_DRIVER_VERSION); + driverConfigUpgrader.upgrade(job.getDriverConfig(), newDriver.getDriverConfig()); + MJob newJob = new MJob(job, job.getFromJobConfig(), job.getToJobConfig(), newDriver.getDriverConfig()); + + // Transform config structures to objects for validations + Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getDriverConfigurationGroupClass()); + ConfigUtils.fromConfigs(newJob.getDriverConfig().getConfigs(), newConfigurationObject); + + ConfigValidator validation = validator.validateConfigForJob(newConfigurationObject); if (validation.getStatus().canProceed()) { updateJob(newJob, tx); } else { @@ -587,14 +547,14 @@ public abstract class Repository { if(tx != null) { tx.close(); } - LOG.info("Driver config upgrade finished"); + LOG.info("Driver upgrade finished"); } } - private void logInvalidModelObject(String objectType, MPersistableEntity entity, Validation validation) { + private void logInvalidModelObject(String objectType, MPersistableEntity entity, ConfigValidator validation) { LOG.error("Upgrader created invalid " + objectType + " with id" + entity.getPersistenceId()); - for(Map.Entry<Validation.FormInput, Validation.Message> entry : validation.getMessages().entrySet()) { + for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : validation.getMessages().entrySet()) { LOG.error("\t" + entry.getKey() + ": " + entry.getValue()); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java index ae7be82..c2f8505 100644 --- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java +++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java @@ -118,7 +118,7 @@ public class RepositoryManager implements Reconfigurable { provider.initialize(context); if(!immutableRepository) { - LOG.info("Creating or upgrading on disk structures if necessary"); + LOG.info("Creating or update respository internals at bootup"); provider.getRepository().createOrUpdateInternals(); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/main/resources/driver-config.properties ---------------------------------------------------------------------- diff --git a/core/src/main/resources/driver-config.properties b/core/src/main/resources/driver-config.properties index 9ec5d9e..78c3b38 100644 --- a/core/src/main/resources/driver-config.properties +++ b/core/src/main/resources/driver-config.properties @@ -19,12 +19,12 @@ # Throttling From # -throttling.label = Throttling resources -throttling.help = Set throttling boundaries to not overload your systems +throttthrottlingConfigling.label = Throttling resources +throttlingConfig.help = Set throttling boundaries to not overload your systems -throttling.extractors.label = Extractors -throttling.extractors.help = Number of extractors that Sqoop will use +throttlingConfig.numExtractors.label = Extractors +throttlingConfig.numExtractors.help = Number of extractors that Sqoop will use -throttling.loaders.label = Loaders -throttling.loaders.help = Number of loaders that Sqoop will use +throttlingConfig.numLoaders.label = Loaders +throttlingConfig.numLoaders.help = Number of loaders that Sqoop will use http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java index 9c39d23..dc4e8c8 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java @@ -18,16 +18,19 @@ */ package org.apache.sqoop.driver; -import org.apache.sqoop.driver.DriverConfigUpgrader; -import org.apache.sqoop.model.*; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import java.util.LinkedList; import java.util.List; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import org.apache.sqoop.model.MConfig; +import org.apache.sqoop.model.MConfigList; +import org.apache.sqoop.model.MInput; +import org.apache.sqoop.model.MIntegerInput; +import org.apache.sqoop.model.MStringInput; +import org.junit.Before; +import org.junit.Test; /** */ @@ -39,35 +42,14 @@ public class TestDriverConfigUpgrader { public void initializeUpgrader() { upgrader = new DriverConfigUpgrader(); } - - /** - * We take the same forms on input and output and we - * expect that all values will be correctly transferred. - */ - @Test - public void testConnectionUpgrade() { - MConnectionForms original = connection1(); - MConnectionForms target = connection1(); - - original.getStringInput("f1.s1").setValue("A"); - original.getStringInput("f1.s2").setValue("B"); - original.getIntegerInput("f1.i").setValue(3); - - upgrader.upgrade(original, target); - - assertEquals("A", target.getStringInput("f1.s1").getValue()); - assertEquals("B", target.getStringInput("f1.s2").getValue()); - assertEquals(3, (long)target.getIntegerInput("f1.i").getValue()); - } - /** - * We take the same forms on input and output and we + * We take the same configs on input and output and we * expect that all values will be correctly transferred. */ @Test - public void testJobUpgrade() { - MJobForms original = job1(); - MJobForms target = job1(); + public void testJobConfigTyeUpgrade() { + MConfigList original = job(); + MConfigList target = job(); original.getStringInput("f1.s1").setValue("A"); original.getStringInput("f1.s2").setValue("B"); @@ -85,8 +67,8 @@ public class TestDriverConfigUpgrader { */ @Test public void testNonExistingInput() { - MConnectionForms original = connection1(); - MConnectionForms target = connection2(); + MConfigList original = job1(); + MConfigList target = job2(); original.getStringInput("f1.s1").setValue("A"); original.getStringInput("f1.s2").setValue("B"); @@ -104,9 +86,9 @@ public class TestDriverConfigUpgrader { * therefore is missing in the original. */ @Test - public void testNonExistingForm() { - MConnectionForms original = connection1(); - MConnectionForms target = connection3(); + public void testNonExistingConfig() { + MConfigList original = job1(); + MConfigList target = job3(); original.getStringInput("f1.s1").setValue("A"); original.getStringInput("f1.s2").setValue("B"); @@ -119,25 +101,25 @@ public class TestDriverConfigUpgrader { assertNull(target.getIntegerInput("f2.i").getValue()); } - MJobForms job1() { - return new MJobForms(forms1()); + MConfigList job() { + return new MConfigList(configs1()); } - MConnectionForms connection1() { - return new MConnectionForms(forms1()); + MConfigList job1() { + return new MConfigList(configs1()); } - MConnectionForms connection2() { - return new MConnectionForms(forms2()); + MConfigList job2() { + return new MConfigList(configs2()); } - MConnectionForms connection3() { - return new MConnectionForms(forms3()); + MConfigList job3() { + return new MConfigList(configs3()); } - List<MForm> forms1() { - List<MForm> list = new LinkedList<MForm>(); - list.add(new MForm("f1", inputs1("f1"))); + List<MConfig> configs1() { + List<MConfig> list = new LinkedList<MConfig>(); + list.add(new MConfig("f1", inputs1("f1"))); return list; } @@ -149,9 +131,9 @@ public class TestDriverConfigUpgrader { return list; } - List<MForm> forms2() { - List<MForm> list = new LinkedList<MForm>(); - list.add(new MForm("f1", inputs2("f1"))); + List<MConfig> configs2() { + List<MConfig> list = new LinkedList<MConfig>(); + list.add(new MConfig("f1", inputs2("f1"))); return list; } @@ -163,9 +145,9 @@ public class TestDriverConfigUpgrader { return list; } - List<MForm> forms3() { - List<MForm> list = new LinkedList<MForm>(); - list.add(new MForm("f2", inputs1("f2"))); + List<MConfig> configs3() { + List<MConfig> list = new LinkedList<MConfig>(); + list.add(new MConfig("f2", inputs1("f2"))); return list; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/8362c73c/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java index aa3af89..3b475c6 100644 --- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java +++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java @@ -20,6 +20,13 @@ package org.apache.sqoop.driver; import java.util.Arrays; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import org.apache.sqoop.common.Direction; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.ConnectorManager; @@ -34,13 +41,6 @@ import org.apache.sqoop.request.HttpEventContext; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class TestJobManager { private JobManager jobManager; private SqoopConnector sqoopConnectorMock; @@ -99,7 +99,7 @@ public class TestJobManager { @Test public void testGetLink() { - MLink testLink = new MLink(123l, null, null); + MLink testLink = new MLink(123l, null); testLink.setEnabled(true); MLink mConnectionSpy = org.mockito.Mockito.spy(testLink); when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock); @@ -111,7 +111,7 @@ public class TestJobManager { @Test public void testDisabledLink() { - MLink testConnection = new MLink(123l, null, null); + MLink testConnection = new MLink(123l, null); testConnection.setPersistenceId(1234); testConnection.setEnabled(false); SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection id: "