SQOOP-1506: Sqoop2: From/To: Re-enable all tests (Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/fcb77b67 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/fcb77b67 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/fcb77b67 Branch: refs/heads/SQOOP-1367 Commit: fcb77b671345fc3b1e6d7b4285acce420f52ec01 Parents: eddd4dd Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Sep 11 10:25:37 2014 +0200 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Sep 11 10:25:37 2014 +0200 ---------------------------------------------------------------------- .../apache/sqoop/client/TestSqoopClient.java | 363 ++-- .../apache/sqoop/model/MValidatedElement.java | 1 + .../validation/validators/NullOrContains.java | 32 + .../validators/TestNullOrContains.java | 97 + .../jdbc/configuration/FromTableForm.java | 4 +- .../sqoop/connector/jdbc/TestToInitializer.java | 30 +- .../apache/sqoop/connector/hdfs/HdfsLoader.java | 3 +- .../hdfs/configuration/StorageType.java | 28 - .../sqoop/connector/hdfs/TestExtractor.java | 25 +- .../sqoop/connector/hdfs/TestHdfsBase.java | 14 +- .../configuration/ConnectionConfiguration.java | 2 +- .../sqoop/framework/TestFrameworkValidator.java | 178 +- .../sqoop/repository/TestJdbcRepository.java | 1854 +++++++++--------- .../org/apache/sqoop/job/mr/SqoopMapper.java | 3 + .../mapreduce/MapreduceExecutionEngineTest.java | 106 - pom.xml | 3 - .../minicluster/TomcatSqoopMiniCluster.java | 1 + .../sqoop/test/testcases/ConnectorTestCase.java | 12 +- .../org/apache/sqoop/test/utils/HdfsUtils.java | 4 +- .../jdbc/generic/FromHDFSToRDBMSTest.java | 78 + .../jdbc/generic/FromRDBMSToHDFSTest.java | 121 ++ .../connector/jdbc/generic/PartitionerTest.java | 128 ++ .../connector/jdbc/generic/TableExportTest.java | 73 - .../connector/jdbc/generic/TableImportTest.java | 108 - .../jdbc/generic/TableStagedRDBMSTest.java | 86 + .../generic/exports/TableStagedExportTest.java | 77 - .../jdbc/generic/imports/PartitionerTest.java | 117 -- .../SubmissionWithDisabledModelObjectsTest.java | 147 +- .../sqoop/integration/server/VersionTest.java | 16 +- 29 files changed, 1848 insertions(+), 1863 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java ---------------------------------------------------------------------- diff --git a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java index b5e7e61..731f6cc 100644 --- a/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java +++ b/client/src/test/java/org/apache/sqoop/client/TestSqoopClient.java @@ -43,185 +43,186 @@ import static org.mockito.Mockito.*; public class TestSqoopClient { -// SqoopRequests requests; -// SqoopClient client; -// -// @Before -// public void setUp() { -// requests = mock(SqoopRequests.class); -// client = new SqoopClient("my-cool-server"); -// client.setSqoopRequests(requests); -// } -// -// /** -// * Retrieve connector information, request to bundle for same connector should -// * not require additional HTTP request. -// */ -// @Test -// public void testGetConnector() { -// when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1))); -// MConnector connector = client.getConnector(1); -// assertEquals(1, connector.getPersistenceId()); -// -// client.getResourceBundle(1L); -// -// verify(requests, times(1)).readConnector(1L); -// } -// -// @Test -// public void testGetConnectorByString() { -// when(requests.readConnector(null)).thenReturn(connectorBean(connector(1))); -// MConnector connector = client.getConnector("A1"); -// assertEquals(1, connector.getPersistenceId()); -// assertEquals("A1", connector.getUniqueName()); -// -// client.getResourceBundle(1L); -// -// verify(requests, times(0)).readConnector(1L); -// verify(requests, times(1)).readConnector(null); -// } -// -// /** -// * Retrieve connector bundle, request for metadata for same connector should -// * not require additional HTTP request. -// */ -// @Test -// public void testGetConnectorBundle() { -// when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1))); -// client.getResourceBundle(1L); -// -// MConnector connector = client.getConnector(1); -// assertEquals(1, connector.getPersistenceId()); -// -// verify(requests, times(1)).readConnector(1L); -// } -// -// /** -// * Retrieve framework information, request to framework bundle should not -// * require additional HTTP request. -// */ -// @Test -// public void testGetFramework() { -// when(requests.readFramework()).thenReturn(frameworkBean(framework())); -// -// client.getFramework(); -// client.getFrameworkResourceBundle(); -// -// verify(requests, times(1)).readFramework(); -// } -// -// /** -// * Retrieve framework bundle, request to framework metadata should not -// * require additional HTTP request. -// */ -// @Test -// public void testGetFrameworkBundle() { -// when(requests.readFramework()).thenReturn(frameworkBean(framework())); -// -// client.getFrameworkResourceBundle(); -// client.getFramework(); -// -// verify(requests, times(1)).readFramework(); -// } -// -// /** -// * Getting all connectors at once should avoid any other HTTP request to -// * specific connectors. -// */ -// @Test -// public void testGetConnectors() { -// MConnector connector; -// -// when(requests.readConnector(null)).thenReturn(connectorBean(connector(1), connector(2))); -// Collection<MConnector> connectors = client.getConnectors(); -// assertEquals(2, connectors.size()); -// -// client.getResourceBundle(1); -// connector = client.getConnector(1); -// assertEquals(1, connector.getPersistenceId()); -// -// connector = client.getConnector(2); -// client.getResourceBundle(2); -// assertEquals(2, connector.getPersistenceId()); -// -// connectors = client.getConnectors(); -// assertEquals(2, connectors.size()); -// -// connector = client.getConnector("A1"); -// assertEquals(1, connector.getPersistenceId()); -// assertEquals("A1", connector.getUniqueName()); -// -// connector = client.getConnector("A2"); -// assertEquals(2, connector.getPersistenceId()); -// assertEquals("A2", connector.getUniqueName()); -// -// connector = client.getConnector("A3"); -// assertNull(connector); -// -// verify(requests, times(1)).readConnector(null); -// verifyNoMoreInteractions(requests); -// } -// -// -// /** -// * Getting connectors one by one should not be equivalent to getting all connectors -// * at once as Client do not know how many connectors server have. -// */ -// @Test -// public void testGetConnectorOneByOne() { -// ConnectorBean bean = connectorBean(connector(1), connector(2)); -// when(requests.readConnector(null)).thenReturn(bean); -// when(requests.readConnector(1L)).thenReturn(bean); -// when(requests.readConnector(2L)).thenReturn(bean); -// -// client.getResourceBundle(1); -// client.getConnector(1); -// -// client.getConnector(2); -// client.getResourceBundle(2); -// -// Collection<MConnector> connectors = client.getConnectors(); -// assertEquals(2, connectors.size()); -// -// verify(requests, times(1)).readConnector(null); -// verify(requests, times(1)).readConnector(1L); -// verify(requests, times(1)).readConnector(2L); -// verifyNoMoreInteractions(requests); -// } -// -// /** -// * Connection for non-existing connector can't be created. -// */ -// @Test(expected = SqoopException.class) -// public void testNewConnection() { -// when(requests.readConnector(null)).thenReturn(connectorBean(connector(1))); -// client.newConnection("non existing connector"); -// } -// -// private ConnectorBean connectorBean(MConnector...connectors) { -// List<MConnector> connectorList = new ArrayList<MConnector>(); -// Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); -// -// for(MConnector connector : connectors) { -// connectorList.add(connector); -// bundles.put(connector.getPersistenceId(), null); -// } -// return new ConnectorBean(connectorList, bundles); -// } -// private FrameworkBean frameworkBean(MFramework framework) { -// return new FrameworkBean(framework, new MapResourceBundle(null)); -// } -// -// private MConnector connector(long id) { -// MConnector connector = new MConnector("A" + id, "A" + id, "1.0" + id, new MConnectionForms(null), new LinkedList<MJobForms>()); -// connector.setPersistenceId(id); -// return connector; -// } -// -// private MFramework framework() { -// MFramework framework = new MFramework(new MConnectionForms(null), -// new LinkedList<MJobForms>(), "1"); -// framework.setPersistenceId(1); -// return framework; -// } + SqoopRequests requests; + SqoopClient client; + + @Before + public void setUp() { + requests = mock(SqoopRequests.class); + client = new SqoopClient("my-cool-server"); + client.setSqoopRequests(requests); + } + + /** + * Retrieve connector information, request to bundle for same connector should + * not require additional HTTP request. + */ + @Test + public void testGetConnector() { + when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1))); + MConnector connector = client.getConnector(1); + assertEquals(1, connector.getPersistenceId()); + + client.getResourceBundle(1L); + + verify(requests, times(1)).readConnector(1L); + } + + @Test + public void testGetConnectorByString() { + when(requests.readConnector(null)).thenReturn(connectorBean(connector(1))); + MConnector connector = client.getConnector("A1"); + assertEquals(1, connector.getPersistenceId()); + assertEquals("A1", connector.getUniqueName()); + + client.getResourceBundle(1L); + + verify(requests, times(0)).readConnector(1L); + verify(requests, times(1)).readConnector(null); + } + + /** + * Retrieve connector bundle, request for metadata for same connector should + * not require additional HTTP request. + */ + @Test + public void testGetConnectorBundle() { + when(requests.readConnector(1L)).thenReturn(connectorBean(connector(1))); + client.getResourceBundle(1L); + + MConnector connector = client.getConnector(1); + assertEquals(1, connector.getPersistenceId()); + + verify(requests, times(1)).readConnector(1L); + } + + /** + * Retrieve framework information, request to framework bundle should not + * require additional HTTP request. + */ + @Test + public void testGetFramework() { + when(requests.readFramework()).thenReturn(frameworkBean(framework())); + + client.getFramework(); + client.getFrameworkResourceBundle(); + + verify(requests, times(1)).readFramework(); + } + + /** + * Retrieve framework bundle, request to framework metadata should not + * require additional HTTP request. + */ + @Test + public void testGetFrameworkBundle() { + when(requests.readFramework()).thenReturn(frameworkBean(framework())); + + client.getFrameworkResourceBundle(); + client.getFramework(); + + verify(requests, times(1)).readFramework(); + } + + /** + * Getting all connectors at once should avoid any other HTTP request to + * specific connectors. + */ + @Test + public void testGetConnectors() { + MConnector connector; + + when(requests.readConnector(null)).thenReturn(connectorBean(connector(1), connector(2))); + Collection<MConnector> connectors = client.getConnectors(); + assertEquals(2, connectors.size()); + + client.getResourceBundle(1); + connector = client.getConnector(1); + assertEquals(1, connector.getPersistenceId()); + + connector = client.getConnector(2); + client.getResourceBundle(2); + assertEquals(2, connector.getPersistenceId()); + + connectors = client.getConnectors(); + assertEquals(2, connectors.size()); + + connector = client.getConnector("A1"); + assertEquals(1, connector.getPersistenceId()); + assertEquals("A1", connector.getUniqueName()); + + connector = client.getConnector("A2"); + assertEquals(2, connector.getPersistenceId()); + assertEquals("A2", connector.getUniqueName()); + + connector = client.getConnector("A3"); + assertNull(connector); + + verify(requests, times(1)).readConnector(null); + verifyNoMoreInteractions(requests); + } + + + /** + * Getting connectors one by one should not be equivalent to getting all connectors + * at once as Client do not know how many connectors server have. + */ + @Test + public void testGetConnectorOneByOne() { + ConnectorBean bean = connectorBean(connector(1), connector(2)); + when(requests.readConnector(null)).thenReturn(bean); + when(requests.readConnector(1L)).thenReturn(bean); + when(requests.readConnector(2L)).thenReturn(bean); + + client.getResourceBundle(1); + client.getConnector(1); + + client.getConnector(2); + client.getResourceBundle(2); + + Collection<MConnector> connectors = client.getConnectors(); + assertEquals(2, connectors.size()); + + verify(requests, times(1)).readConnector(null); + verify(requests, times(1)).readConnector(1L); + verify(requests, times(1)).readConnector(2L); + verifyNoMoreInteractions(requests); + } + + /** + * Connection for non-existing connector can't be created. + */ + @Test(expected = SqoopException.class) + public void testNewConnection() { + when(requests.readConnector(null)).thenReturn(connectorBean(connector(1))); + client.newConnection("non existing connector"); + } + + private ConnectorBean connectorBean(MConnector...connectors) { + List<MConnector> connectorList = new ArrayList<MConnector>(); + Map<Long, ResourceBundle> bundles = new HashMap<Long, ResourceBundle>(); + + for(MConnector connector : connectors) { + connectorList.add(connector); + bundles.put(connector.getPersistenceId(), null); + } + return new ConnectorBean(connectorList, bundles); + } + private FrameworkBean frameworkBean(MFramework framework) { + return new FrameworkBean(framework, new MapResourceBundle(null)); + } + + private MConnector connector(long id) { + MConnector connector = new MConnector("A" + id, "A" + id, "1.0" + id, + new MConnectionForms(null), new MJobForms(null), new MJobForms(null)); + connector.setPersistenceId(id); + return connector; + } + + private MFramework framework() { + MFramework framework = new MFramework(new MConnectionForms(null), + new MJobForms(null), "1"); + framework.setPersistenceId(1); + return framework; + } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java b/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java index a50c815..c0d678a 100644 --- a/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java +++ b/common/src/main/java/org/apache/sqoop/model/MValidatedElement.java @@ -45,6 +45,7 @@ public abstract class MValidatedElement extends MNamedElement { public MValidatedElement(MValidatedElement other) { super(other); + resetValidationMessages(); this.validationStatus = other.validationStatus; this.validationMessages.addAll(other.validationMessages); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/common/src/main/java/org/apache/sqoop/validation/validators/NullOrContains.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/sqoop/validation/validators/NullOrContains.java b/common/src/main/java/org/apache/sqoop/validation/validators/NullOrContains.java new file mode 100644 index 0000000..9d11a2e --- /dev/null +++ b/common/src/main/java/org/apache/sqoop/validation/validators/NullOrContains.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.validation.validators; + +import org.apache.sqoop.validation.Status; + +/** + * String validator to verify presence of a sub string (provided as a String argument) + */ +public class NullOrContains extends AbstractValidator<String> { + @Override + public void validate(String str) { + if(str != null && !str.contains(getStringArgument())) { + addMessage(Status.UNACCEPTABLE, "String must contain substring: " + getStringArgument()); + } + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java b/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java new file mode 100644 index 0000000..4bd1559 --- /dev/null +++ b/common/src/test/java/org/apache/sqoop/validation/validators/TestNullOrContains.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.validation.validators; + +import org.apache.sqoop.validation.Status; +import org.junit.Test; + +import static junit.framework.Assert.assertEquals; + +/** + */ +public class TestNullOrContains { + + AbstractValidator validator = new NullOrContains(); + + @Test + public void test() { + assertEquals(0, validator.getMessages().size()); + + // Default, no string argument set + validator.validate("str"); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // Searched substring is entire string + validator.reset(); + validator.setStringArgument("str"); + validator.validate("str"); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // Just starts with + validator.reset(); + validator.setStringArgument("str"); + validator.validate("straaaaa"); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // Ends with + validator.reset(); + validator.setStringArgument("str"); + validator.validate("aaastr"); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // In the middle + validator.reset(); + validator.setStringArgument("str"); + validator.validate("aaastraaa"); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // Repitition + validator.reset(); + validator.setStringArgument("str"); + validator.validate("aaastraaastraaa"); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // Null string + validator.reset(); + validator.setStringArgument("str"); + validator.validate(null); + assertEquals(Status.FINE, validator.getStatus()); + assertEquals(0, validator.getMessages().size()); + + // Empty string + validator.reset(); + validator.setStringArgument("str"); + validator.validate(""); + assertEquals(Status.UNACCEPTABLE, validator.getStatus()); + assertEquals(1, validator.getMessages().size()); + + // "Random" string + validator.reset(); + validator.setStringArgument("str"); + validator.validate("Ahoj tady je meduza"); + assertEquals(Status.UNACCEPTABLE, validator.getStatus()); + assertEquals(1, validator.getMessages().size()); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java index ac72546..72902a2 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/FromTableForm.java @@ -23,7 +23,7 @@ import org.apache.sqoop.model.Input; import org.apache.sqoop.model.Validator; import org.apache.sqoop.validation.Status; import org.apache.sqoop.validation.validators.AbstractValidator; -import org.apache.sqoop.validation.validators.Contains; +import org.apache.sqoop.validation.validators.NullOrContains; /** * @@ -36,7 +36,7 @@ public class FromTableForm { @Input(size = 50) public String tableName; - @Input(size = 2000, validators = {@Validator(value = Contains.class, strArg = GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)}) + @Input(size = 2000, validators = {@Validator(value = NullOrContains.class, strArg = GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN)}) public String sql; @Input(size = 50) http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java index 4831cf8..eb6fcf1 100644 --- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java +++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java @@ -27,6 +27,8 @@ import org.apache.sqoop.job.etl.Initializer; import org.apache.sqoop.job.etl.InitializerContext; import org.apache.sqoop.validation.Status; import org.apache.sqoop.validation.Validation; +import org.apache.sqoop.validation.ValidationResult; +import org.apache.sqoop.validation.ValidationRunner; public class TestToInitializer extends TestCase { private final String schemaName; @@ -274,23 +276,23 @@ public class TestToInitializer extends TestCase { // the stage table jobConf.toTable.tableName = schemalessTableName; jobConf.toTable.clearStageTable = false; - GenericJdbcValidator validator = new GenericJdbcValidator(); - Validation validation = validator.validateJob(jobConf); + ValidationRunner validationRunner = new ValidationRunner(); + ValidationResult result = validationRunner.validate(jobConf); assertEquals("User should not specify clear stage table flag without " + "specifying name of the stage table", Status.UNACCEPTABLE, - validation.getStatus()); - assertTrue(validation.getMessages().containsKey( - new Validation.FormInput("toTable"))); + result.getStatus()); + assertTrue(result.getMessages().containsKey( + "toTable")); jobConf.toTable.clearStageTable = true; - validation = validator.validateJob(jobConf); + result = validationRunner.validate(jobConf); assertEquals("User should not specify clear stage table flag without " + "specifying name of the stage table", Status.UNACCEPTABLE, - validation.getStatus()); - assertTrue(validation.getMessages().containsKey( - new Validation.FormInput("toTable"))); + result.getStatus()); + assertTrue(result.getMessages().containsKey( + "toTable")); } @SuppressWarnings("unchecked") @@ -304,12 +306,12 @@ public class TestToInitializer extends TestCase { jobConf.toTable.stageTableName = stageTableName; jobConf.toTable.sql = ""; - GenericJdbcValidator validator = new GenericJdbcValidator(); - Validation validation = validator.validateJob(jobConf); + ValidationRunner validationRunner = new ValidationRunner(); + ValidationResult result = validationRunner.validate(jobConf); assertEquals("Stage table name cannot be specified without specifying " + - "table name", Status.UNACCEPTABLE, validation.getStatus()); - assertTrue(validation.getMessages().containsKey( - new Validation.FormInput("toTable"))); + "table name", Status.UNACCEPTABLE, result.getStatus()); + assertTrue(result.getMessages().containsKey( + "toTable")); } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java index 5a924f9..61676fe 100644 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java +++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsLoader.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.sqoop.common.PrefixContext; import org.apache.sqoop.common.SqoopException; import org.apache.sqoop.connector.hdfs.configuration.ConnectionConfiguration; import org.apache.sqoop.connector.hdfs.configuration.OutputFormat; @@ -50,7 +51,7 @@ public class HdfsLoader extends Loader<ConnectionConfiguration, ToJobConfigurati DataReader reader = context.getDataReader(); - Configuration conf = new Configuration(); + Configuration conf = ((PrefixContext)context.getContext()).getConfiguration(); String directoryName = job.output.outputDirectory; String codecname = getCompressionCodecName(job); http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/StorageType.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/StorageType.java b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/StorageType.java deleted file mode 100644 index d4aaa0a..0000000 --- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/StorageType.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.sqoop.connector.hdfs.configuration; - -/** - * Various storage types that Sqoop is supporting - */ -public enum StorageType { - /** - * Direct HDFS import - */ - HDFS, -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java index 6ed4087..5e21543 100644 --- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java +++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestExtractor.java @@ -97,9 +97,8 @@ public class TestExtractor extends TestHdfsBase { public void testExtractor() throws Exception { Configuration conf = new Configuration(); PrefixContext prefixContext = new PrefixContext(conf, "org.apache.sqoop.job.connector.from.context."); + final boolean[] visited = new boolean[NUMBER_OF_FILES * NUMBER_OF_ROWS_PER_FILE]; ExtractorContext context = new ExtractorContext(prefixContext, new DataWriter() { - private long index = 1L; - @Override public void writeArrayRecord(Object[] array) { throw new AssertionError("Should not be writing array."); @@ -107,7 +106,23 @@ public class TestExtractor extends TestHdfsBase { @Override public void writeStringRecord(String text) { - Assert.assertEquals(index + "," + index + ".0,'" + index++ + "'", text); + int index; + String[] components = text.split(","); + Assert.assertEquals(3, components.length); + + // Value should take on the form <integer>,<float>,'<integer>' + // for a single index. IE: 1,1.0,'1'. + try { + index = Integer.parseInt(components[0]); + } catch (NumberFormatException e) { + throw new AssertionError("Could not parse int for " + components[0]); + } + + Assert.assertFalse(visited[index - 1]); + Assert.assertEquals(String.valueOf((double)index), components[1]); + Assert.assertEquals("'" + index + "'", components[2]); + + visited[index - 1] = true; } @Override @@ -121,5 +136,9 @@ public class TestExtractor extends TestHdfsBase { HdfsPartition partition = createPartition(FileUtils.listDir(inputDirectory)); extractor.extract(context, connConf, jobConf, partition); + + for (int index = 0; index < NUMBER_OF_FILES * NUMBER_OF_ROWS_PER_FILE; ++index) { + Assert.assertTrue("Index " + (index + 1) + " was not visited", visited[index]); + } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java ---------------------------------------------------------------------- diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java index 0cc2b8b..6eae7fd 100644 --- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java +++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestHdfsBase.java @@ -30,6 +30,7 @@ import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; +import java.util.UUID; public class TestHdfsBase { @@ -66,7 +67,7 @@ public class TestHdfsBase { int index = 1; for (int fi = 0; fi < numberOfFiles; fi++) { - String fileName = indir + "/" + "part-r-" + padZeros(fi, 5) + extension; + String fileName = indir + "/" + UUID.randomUUID() + extension; OutputStream filestream = FileUtils.create(fileName); BufferedWriter filewriter; if (codec != null) { @@ -105,8 +106,7 @@ public class TestHdfsBase { int index = 1; for (int fi = 0; fi < numberOfFiles; fi++) { - Path filepath = new Path(indir, - "part-r-" + padZeros(fi, 5) + ".seq"); + Path filepath = new Path(indir,UUID.randomUUID() + ".seq"); SequenceFile.Writer filewriter; if (codec != null) { filewriter = SequenceFile.createWriter(filepath.getFileSystem(conf), @@ -128,12 +128,4 @@ public class TestHdfsBase { filewriter.close(); } } - - private String padZeros(int number, int digits) { - String string = String.valueOf(number); - for (int i = (digits - string.length()); i > 0; i--) { - string = "0" + string; - } - return string; - } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java b/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java index 830606a..897d3c7 100644 --- a/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java +++ b/core/src/main/java/org/apache/sqoop/framework/configuration/ConnectionConfiguration.java @@ -26,5 +26,5 @@ import org.apache.sqoop.model.Form; @ConfigurationClass public class ConnectionConfiguration { - @Form SecurityForm security; + @Form SecurityForm security = new SecurityForm(); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/fcb77b67/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java index 90395ac..69c1b56 100644 --- a/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java +++ b/core/src/test/java/org/apache/sqoop/framework/TestFrameworkValidator.java @@ -17,7 +17,13 @@ */ package org.apache.sqoop.framework; -//import org.apache.sqoop.framework.configuration.OutputCompression; +import org.apache.sqoop.framework.configuration.ConnectionConfiguration; +import org.apache.sqoop.framework.configuration.JobConfiguration; +import org.apache.sqoop.validation.Status; +import org.apache.sqoop.validation.ValidationResult; +import org.apache.sqoop.validation.ValidationRunner; +import org.junit.Before; +import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -27,134 +33,48 @@ import static org.junit.Assert.assertTrue; */ public class TestFrameworkValidator { -// FrameworkValidator validator; -// -// @Before -// public void setUp() { -// validator = new FrameworkValidator(); -// } -// -// @Test -// public void testConnectionValidation() { -// ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration(); -// -// Validation validation = validator.validateConnection(connectionConfiguration); -// assertEquals(Status.FINE, validation.getStatus()); -// assertEquals(0, validation.getMessages().size()); -// } -// -// @Test -// public void testExportJobValidation() { -// ExportJobConfiguration configuration; -// Validation validation; -// -// // Empty form is not allowed -// configuration = new ExportJobConfiguration(); -// validation = validator.validateJob(MJob.Type.EXPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("input.inputDirectory"))); -// -// // Explicitly setting extractors and loaders -// configuration = new ExportJobConfiguration(); -// configuration.input.inputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 3; -// configuration.throttling.loaders = 3; -// -// validation = validator.validateJob(MJob.Type.EXPORT, configuration); -// assertEquals(Status.FINE, validation.getStatus()); -// assertEquals(0, validation.getMessages().size()); -// -// // Negative and zero values for extractors and loaders -// configuration = new ExportJobConfiguration(); -// configuration.input.inputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 0; -// configuration.throttling.loaders = -1; -// -// validation = validator.validateJob(MJob.Type.EXPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors"))); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders"))); -// } -// -// -// @Test -// public void testImportJobValidation() { -// ImportJobConfiguration configuration; -// Validation validation; -// -// // Empty form is not allowed -// configuration = new ImportJobConfiguration(); -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.outputDirectory"))); -// -// // Explicitly setting extractors and loaders -// configuration = new ImportJobConfiguration(); -// configuration.output.outputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 3; -// configuration.throttling.loaders = 3; -// -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.FINE, validation.getStatus()); -// assertEquals(0, validation.getMessages().size()); -// -// // Negative and zero values for extractors and loaders -// configuration = new ImportJobConfiguration(); -// configuration.output.outputDirectory = "/czech/republic"; + FrameworkValidator validator; + + @Before + public void setUp() { + validator = new FrameworkValidator(); + } + + @Test + public void testConnectionValidation() { + ConnectionConfiguration connectionConfiguration = new ConnectionConfiguration(); + ValidationRunner runner = new ValidationRunner(); + ValidationResult result = runner.validate(connectionConfiguration); + assertEquals(Status.FINE, result.getStatus()); + assertEquals(0, result.getMessages().size()); + } + + @Test + public void testJobValidation() { + ValidationRunner runner = new ValidationRunner(); + ValidationResult result; + JobConfiguration configuration; + + // Empty form is allowed + configuration = new JobConfiguration(); + result = runner.validate(configuration); + assertEquals(Status.FINE, result.getStatus()); + + // Explicitly setting extractors and loaders + configuration = new JobConfiguration(); + configuration.throttling.extractors = 3; + configuration.throttling.loaders = 3; + result = runner.validate(configuration); + assertEquals(Status.FINE, result.getStatus()); + assertEquals(0, result.getMessages().size()); + + // Negative and zero values for extractors and loaders +// configuration = new JobConfiguration(); // configuration.throttling.extractors = 0; // configuration.throttling.loaders = -1; -// -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.extractors"))); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("throttling.loaders"))); -// -// // specifying both compression as well as customCompression is -// // unacceptable -// configuration = new ImportJobConfiguration(); -// configuration.output.outputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 2; -// configuration.throttling.loaders = 2; -// configuration.output.compression = OutputCompression.BZIP2; -// configuration.output.customCompression = "some.compression.codec"; -// -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression"))); -// -// // specifying a customCompression is fine -// configuration = new ImportJobConfiguration(); -// configuration.output.outputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 2; -// configuration.throttling.loaders = 2; -// configuration.output.compression = OutputCompression.CUSTOM; -// configuration.output.customCompression = "some.compression.codec"; -// -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.FINE, validation.getStatus()); -// -// // specifying a customCompression without codec name is unacceptable -// configuration = new ImportJobConfiguration(); -// configuration.output.outputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 2; -// configuration.throttling.loaders = 2; -// configuration.output.compression = OutputCompression.CUSTOM; -// configuration.output.customCompression = ""; -// -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression"))); -// -// configuration = new ImportJobConfiguration(); -// configuration.output.outputDirectory = "/czech/republic"; -// configuration.throttling.extractors = 2; -// configuration.throttling.loaders = 2; -// configuration.output.compression = OutputCompression.CUSTOM; -// configuration.output.customCompression = null; -// -// validation = validator.validateJob(MJob.Type.IMPORT, configuration); -// assertEquals(Status.UNACCEPTABLE, validation.getStatus()); -// assertTrue(validation.getMessages().containsKey(new Validation.FormInput("output.compression"))); -// -// } +// result = runner.validate(configuration); +// assertEquals(Status.FINE, result.getStatus()); +// assertTrue(result.getMessages().containsKey("throttling.extractors")); +// assertTrue(result.getMessages().containsKey("throttling.loaders")); + } }
