Repository: sqoop Updated Branches: refs/heads/trunk 3233db8e1 -> b2643d509
SQOOP-3331: Add Mainframe FTP integration test for GDG dataset. (Chris Teoh via Szabolcs Vasas) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b2643d50 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b2643d50 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b2643d50 Branch: refs/heads/trunk Commit: b2643d5094da405acfc695d1da22a54d32aa74ed Parents: 3233db8 Author: Szabolcs Vasas <[email protected]> Authored: Fri Jun 15 16:42:43 2018 +0200 Committer: Szabolcs Vasas <[email protected]> Committed: Fri Jun 15 16:42:43 2018 +0200 ---------------------------------------------------------------------- build.xml | 15 ++ .../apache/sqoop/manager/MainframeManager.java | 1 - .../sqoop-thirdpartytest-db-services.yml | 11 ++ .../mainframe/MainframeManagerImportTest.java | 141 +++++++++++++++++++ .../manager/mainframe/MainframeTestUtil.java | 44 ++++++ 5 files changed, 211 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/b2643d50/build.xml ---------------------------------------------------------------------- diff --git a/build.xml b/build.xml index a85705f..0ae729b 100644 --- a/build.xml +++ b/build.xml @@ -246,6 +246,13 @@ <property name="sqoop.test.netezza.db.name" value="SQOOP" /> <property name="sqoop.test.netezza.table.name" value="EMPNZ" /> + <property name="sqoop.test.mainframe.ftp.host" value="localhost" /> + <property name="sqoop.test.mainframe.ftp.port" value="2121" /> + <property name="sqoop.test.mainframe.ftp.username" value="test" /> + <property name="sqoop.test.mainframe.ftp.password" value="test" /> + <property name="sqoop.test.mainframe.ftp.dataset.gdg" value="TSODIQ1.GDGTEXT" /> + <property name="sqoop.test.mainframe.ftp.dataset.gdg.filename" value="G0001V43" /> + <property name="sqoop.test.mainframe.ftp.dataset.gdg.md5" value="f0d0d171fdb8a03dbc1266ed179d7093" /> <condition property="windows"> <os family="windows" /> @@ -873,6 +880,14 @@ <sysproperty key="sqoop.test.netezza.db.name" value="${sqoop.test.netezza.db.name}" /> <sysproperty key="sqoop.test.netezza.table.name" value="${sqoop.test.netezza.table.name}" /> + <sysproperty key="sqoop.test.mainframe.ftp.host" value="${sqoop.test.mainframe.ftp.host}" /> + <sysproperty key="sqoop.test.mainframe.ftp.port" value="${sqoop.test.mainframe.ftp.port}" /> + <sysproperty key="sqoop.test.mainframe.ftp.username" value="${sqoop.test.mainframe.ftp.username}" /> + <sysproperty key="sqoop.test.mainframe.ftp.password" value="${sqoop.test.mainframe.ftp.password}" /> + <sysproperty key="sqoop.test.mainframe.ftp.dataset.gdg" value="${sqoop.test.mainframe.ftp.dataset.gdg}" /> + <sysproperty key="sqoop.test.mainframe.ftp.dataset.gdg.filename" value="${sqoop.test.mainframe.ftp.dataset.gdg.filename}" /> + <sysproperty key="sqoop.test.mainframe.ftp.dataset.gdg.md5" value="${sqoop.test.mainframe.ftp.dataset.gdg.md5}" /> + <!-- Location of Hive logs --> <!--<sysproperty key="hive.log.dir" value="${test.build.data}/sqoop/logs"/> --> http://git-wip-us.apache.org/repos/asf/sqoop/blob/b2643d50/src/java/org/apache/sqoop/manager/MainframeManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/MainframeManager.java b/src/java/org/apache/sqoop/manager/MainframeManager.java index 4e8be15..70b9fc9 100644 --- a/src/java/org/apache/sqoop/manager/MainframeManager.java +++ b/src/java/org/apache/sqoop/manager/MainframeManager.java @@ -46,7 +46,6 @@ import org.apache.sqoop.util.ImportException; */ public class MainframeManager extends org.apache.sqoop.manager.ConnManager { public static final String DEFAULT_DATASET_COLUMN_NAME = "DEFAULT_COLUMN"; - protected SqoopOptions options; private static final Log LOG = LogFactory.getLog(MainframeManager.class.getName()); http://git-wip-us.apache.org/repos/asf/sqoop/blob/b2643d50/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml ---------------------------------------------------------------------- diff --git a/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml b/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml index 2f4a07f..4648f54 100644 --- a/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml +++ b/src/scripts/thirdpartytest/docker-compose/sqoop-thirdpartytest-db-services.yml @@ -111,3 +111,14 @@ services: interval: 30s timeout: 10s retries: 20 + mainframe: + image: cntroversycubed/sqoopgdg:afdf57b15d8e71eb77d24d606b77e185ef39ceb3 + container_name: sqoop_mainframe_gdg_container + ports: + - 2121:2121 + - 60000-60009:60000-60009 + healthcheck: + test: ["CMD", "nc", "-z", "localhost", "2121"] + interval: 30s + timeout: 10s + retries: 20 http://git-wip-us.apache.org/repos/asf/sqoop/blob/b2643d50/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java b/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java new file mode 100644 index 0000000..041dfb7 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/mainframe/MainframeManagerImportTest.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.mainframe; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Set; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.sqoop.mapreduce.mainframe.MainframeConfiguration; +import org.apache.sqoop.testutil.CommonArgs; +import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.tool.MainframeImportTool; +import org.apache.sqoop.util.FileListing; +import org.apache.commons.lang3.StringUtils; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Mainframe integration tests + * Can be run using -Dtestcase=MainframeManagerImportTest or -Dthirdparty=true. + * To run tests:- + * Start docker containers by running start-thirdpartytest-db-containers.sh + * Run test using ant test -Dtestcase=MainframeManagerImportTest or -Dthirdparty=true + * Clean up containers by running stop-thirdpartytest-db-containers.sh + * The following properties can be overridden from command line + * by setting -D<property.name>=<value>:- + * <property name="sqoop.test.mainframe.ftp.host" value="localhost" /> + * <property name="sqoop.test.mainframe.ftp.port" value="2121" /> + * <property name="sqoop.test.mainframe.ftp.username" value="test" /> + * <property name="sqoop.test.mainframe.ftp.password" value="test" /> + * <property name="sqoop.test.mainframe.ftp.dataset.gdg" value="TSODIQ1.FOLDER" /> + * <property name="sqoop.test.mainframe.ftp.dataset.gdg.filename" value="G0001V43" /> + * <property name="sqoop.test.mainframe.ftp.dataset.gdg.md5" value="43eefbe34e466dd3f65a3e867a60809a" /> + */ + +public class MainframeManagerImportTest extends ImportJobTestCase { + private static final Log LOG = LogFactory.getLog( + MainframeManagerImportTest.class.getName()); + + @Override + protected boolean useHsqldbTestServer() { + return false; + } + + /** Does the import and verify + * @param datasetName the mainframe dataset name + * @param datasetType the mainframe dataset type from MainframeConfiguration (s/g/p) + * @param fileHashes each HashMap entry is filename, expected md5sum + * @param extraArgs extra arguments to the tool if required + * @throws IOException if it fails to delete the directory or read the file + * @throws RuntimeException if it fails to run the mainframe import + */ + private void doImportAndVerify(String datasetName, String datasetType, HashMap<String,String> fileHashes, String ... extraArgs) throws IOException, RuntimeException { + Path tablePath = new Path(datasetName); + + File tableFile = new File(tablePath.toString()); + if (tableFile.exists() && tableFile.isDirectory()) { + // remove the directory before running the import. + LOG.info(String.format("Removing folder: %s", tableFile)); + FileListing.recursiveDeleteDir(tableFile); + } + + String [] argv = getArgv(datasetName, datasetType, extraArgs); + try { + MainframeImportTool tool = new MainframeImportTool(); + runImport(tool,argv); + } catch (IOException ioe) { + LOG.error("Got IOException during import: " + ioe); + throw new RuntimeException(ioe); + } + + Set<String> keys = fileHashes.keySet(); + for (String i : keys) { + Path filePath = new Path(tablePath, String.format("%s%s", i,"-m-00000")); + LOG.info(String.format("Checking for presence of file: %s with MD5 hash %s",filePath,fileHashes.get(i))); + File f = new File(filePath.toString()); + assertTrue("Could not find imported data file", f.exists()); + FileInputStream fis = new FileInputStream(f); + String md5 = DigestUtils.md5Hex(fis); + fis.close(); + assertTrue(String.format("MD5 sums do not match for file: %s. Got MD5 of %s and expected %s",filePath,md5,fileHashes.get(i)),StringUtils.equalsIgnoreCase(md5, fileHashes.get(i))); + } + } + + @Test + public void testImportGdgText() throws IOException { + HashMap<String,String> files = new HashMap<String,String>(); + files.put(MainframeTestUtil.GDG_DATASET_FILENAME, MainframeTestUtil.EXPECTED_GDG_DATASET_MD5); + doImportAndVerify(MainframeTestUtil.GDG_DATASET_NAME, MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_GDG, files); + } + + private String [] getArgv(String datasetName, String datasetType, String ... extraArgs) { + ArrayList<String> args = new ArrayList<String>(); + + CommonArgs.addHadoopFlags(args); + + args.add("--connect"); + args.add(String.format("%s:%s", MainframeTestUtil.HOST, MainframeTestUtil.PORT)); + args.add("--username"); + args.add(MainframeTestUtil.USERNAME); + args.add("--password"); + args.add(MainframeTestUtil.PASSWORD); + args.add("--dataset"); + args.add(datasetName); + args.add("--datasettype"); + args.add(datasetType); + + if (extraArgs.length > 0) { + args.add("--"); + for (String arg : extraArgs) { + args.add(arg); + } + } + + return args.toArray(new String[0]); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/b2643d50/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java b/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java new file mode 100644 index 0000000..f28ff36 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/mainframe/MainframeTestUtil.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.mainframe; + +public class MainframeTestUtil { + public static final String HOST = System.getProperty( + "sqoop.test.mainframe.ftp.host", + "localhost"); + public static final String PORT = System.getProperty( + "sqoop.test.mainframe.ftp.port", + "2121"); + public static final String USERNAME = System.getProperty( + "sqoop.test.mainframe.ftp.username", + "test"); + public static final String PASSWORD = System.getProperty( + "sqoop.test.mainframe.ftp.password", + "test"); + public static final String GDG_DATASET_NAME = System.getProperty( + "sqoop.test.mainframe.ftp.dataset.gdg", + "TSODIQ1.GDGTEXT"); + public static final String GDG_DATASET_FILENAME = System.getProperty( + "sqoop.test.mainframe.ftp.dataset.gdg.filename", + "G0001V43" + ); + public static final String EXPECTED_GDG_DATASET_MD5 = System.getProperty( + "sqoop.test.mainframe.ftp.dataset.gdg.md5", + "f0d0d171fdb8a03dbc1266ed179d7093"); +}
