http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java b/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java new file mode 100644 index 0000000..c4caafb --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseImportAddRowKeyTest.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import org.apache.commons.lang.StringUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static java.util.Arrays.asList; +import static org.apache.commons.lang.StringUtils.join; + +@RunWith(Parameterized.class) +public class HBaseImportAddRowKeyTest extends HBaseTestCase { + + @Parameterized.Parameters(name = "bulkLoad = {0}") + public static Iterable<? extends Object> bulkLoadParameters() { + return Arrays.asList(new Boolean[] { false } , new Boolean[] { true } ); + } + + private String[] columnTypes; + + private String[] columnValues; + + private String hbaseTableName; + + private String hbaseColumnFamily; + + private String hbaseTmpDir; + + private String hbaseBulkLoadDir; + + private boolean bulkLoad; + + public HBaseImportAddRowKeyTest(boolean bulkLoad) { + this.bulkLoad = bulkLoad; + } + + @Before + public void setUp() { + super.setUp(); + columnTypes = new String[] { "INT", "INT" }; + columnValues = new String[] { "0", "1" }; + hbaseTableName = "addRowKeyTable"; + hbaseColumnFamily = "addRowKeyFamily"; + hbaseTmpDir = TEMP_BASE_DIR + "hbaseTmpDir"; + hbaseBulkLoadDir = TEMP_BASE_DIR + "hbaseBulkLoadDir"; + createTableWithColTypes(columnTypes, columnValues); + } + + @Test + public void testAddRowKey() throws IOException { + String[] argv = getImportArguments(true, hbaseTableName, hbaseColumnFamily); + + runImport(argv); + + // Row key should have been added + verifyHBaseCell(hbaseTableName, columnValues[0], hbaseColumnFamily, getColName(0), columnValues[0]); + verifyHBaseCell(hbaseTableName, columnValues[0], hbaseColumnFamily, getColName(1), columnValues[1]); + } + + @Test + public void testAddRowKeyDefault() throws IOException { + String[] argv = getImportArguments(false, hbaseTableName, hbaseColumnFamily); + + runImport(argv); + + // Row key should not be added by default + verifyHBaseCell(hbaseTableName, columnValues[0], hbaseColumnFamily, getColName(0), null); + verifyHBaseCell(hbaseTableName, columnValues[0], hbaseColumnFamily, getColName(1), columnValues[1]); + } + + @Test + public void testAddCompositeKey() throws IOException { + String rowKey = getColName(0)+","+getColName(1); + + String[] argv = getImportArguments(true, hbaseTableName, hbaseColumnFamily, rowKey); + + runImport(argv); + + // Row key should have been added + verifyHBaseCell(hbaseTableName, join(columnValues, '_'), hbaseColumnFamily, getColName(0), columnValues[0]); + verifyHBaseCell(hbaseTableName, join(columnValues, '_'), hbaseColumnFamily, getColName(1), columnValues[1]); + } + + private String[] getImportArguments(boolean addRowKey, String hbaseTableName, String hbaseColumnFamily) { + return getImportArguments(addRowKey, hbaseTableName, hbaseColumnFamily, null); + } + + private String[] getImportArguments(boolean addRowKey, String hbaseTableName, String hbaseColumnFamily, String rowKey) { + List<String> result = new ArrayList<>(); + + if (addRowKey) { + result.add("-D"); + result.add("sqoop.hbase.add.row.key=true"); + } + result.add("-D"); + result.add("hbase.fs.tmp.dir=" + hbaseTmpDir); + + result.addAll(asList(getArgv(true, hbaseTableName, hbaseColumnFamily, true, null))); + + if(bulkLoad) { + result.add("--target-dir"); + result.add(hbaseBulkLoadDir); + result.add("--hbase-bulkload"); + } + + if (!StringUtils.isBlank(rowKey)) { + result.add("--hbase-row-key"); + result.add(rowKey); + } + + return result.toArray(new String[result.size()]); + } + +}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseImportNullTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseImportNullTest.java b/src/test/org/apache/sqoop/hbase/HBaseImportNullTest.java new file mode 100644 index 0000000..114cc6a --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseImportNullTest.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import java.io.IOException; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * + */ +public class HBaseImportNullTest extends HBaseTestCase { + + @Test + public void testNullRow() throws IOException { + String [] argv = getArgv(true, "nullRowT", "nullRowF", true, null); + String [] types = { "INT", "INT" }; + String [] vals = { "0", "null" }; + createTableWithColTypes(types, vals); + runImport(argv); + + // This cell should not be placed in the results.. + verifyHBaseCell("nullRowT", "0", "nullRowF", getColName(1), null); + + int rowCount = countHBaseTable("nullRowT", "nullRowF"); + assertEquals(0, rowCount); + } + + @Test + public void testNulls() throws IOException { + String [] argv = getArgv(true, "nullT", "nullF", true, null); + String [] types = { "INT", "INT", "INT" }; + String [] vals = { "0", "42", "null" }; + createTableWithColTypes(types, vals); + runImport(argv); + + // This cell should import correctly. + verifyHBaseCell("nullT", "0", "nullF", getColName(1), "42"); + + // This cell should not be placed in the results.. + verifyHBaseCell("nullT", "0", "nullF", getColName(2), null); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseImportTest.java b/src/test/org/apache/sqoop/hbase/HBaseImportTest.java new file mode 100644 index 0000000..2e73cf3 --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseImportTest.java @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import java.io.IOException; + +import org.junit.Test; + +import static org.junit.Assert.fail; + +/** + * Test imports of tables into HBase. + */ +public class HBaseImportTest extends HBaseTestCase { + + @Test + public void testBasicUsage() throws IOException { + // Create the HBase table in Sqoop as we run the job. + String [] argv = getArgv(true, "BasicUsage", "BasicColFam", true, null); + String [] types = { "INT", "INT" }; + String [] vals = { "0", "1" }; + createTableWithColTypes(types, vals); + runImport(argv); + verifyHBaseCell("BasicUsage", "0", "BasicColFam", getColName(1), "1"); + } + + @Test + public void testMissingTableFails() throws IOException { + // Test that if the table doesn't exist, we fail unless we + // explicitly create the table. + String [] argv = getArgv(true, "MissingTable", "MissingFam", false, null); + String [] types = { "INT", "INT" }; + String [] vals = { "0", "1" }; + createTableWithColTypes(types, vals); + try { + runImport(argv); + fail("Expected IOException"); + } catch (IOException ioe) { + LOG.info("Got exception -- ok; we expected that job to fail."); + } + } + + @Test + public void testOverwriteSucceeds() throws IOException { + // Test that we can create a table and then import immediately + // back on top of it without problem. + String [] argv = getArgv(true, "OverwriteT", "OverwriteF", true, null); + String [] types = { "INT", "INT" }; + String [] vals = { "0", "1" }; + createTableWithColTypes(types, vals); + runImport(argv); + verifyHBaseCell("OverwriteT", "0", "OverwriteF", getColName(1), "1"); + // Run a second time. + runImport(argv); + verifyHBaseCell("OverwriteT", "0", "OverwriteF", getColName(1), "1"); + } + + @Test + public void testOverwriteNullColumnsSucceeds() throws IOException { + // Test that we can create a table and then import immediately + // back on top of it without problem and then update with null to validate + String [] argv = getArgv(true, "OverwriteTable", "OverwriteColumnFamily", true, null); + String [] types = { "INT", "INT", "INT", "DATETIME" }; + String [] vals = { "0", "1", "1", "'2017-03-20'" }; + createTableWithColTypes(types, vals); + runImport(argv); + verifyHBaseCell("OverwriteTable", "0", "OverwriteColumnFamily", getColName(2), "1"); + // Run a second time. + argv = getIncrementalArgv(true, "OverwriteTable", "OverwriteColumnFamily", true, null, false, false, "DATA_COL3", "2017-03-24 01:01:01.0", null); + vals = new String[] { "0", "1", null, "'2017-03-25'" }; + updateTable(types, vals); + runImport(argv); + verifyHBaseCell("OverwriteTable", "0", "OverwriteColumnFamily", getColName(2), null); + } + + @Test + public void testAppendWithTimestampSucceeds() throws IOException { + // Test that we can create a table and then import multiple rows + // validate for append scenario with time stamp + String [] argv = getArgv(true, "AppendTable", "AppendColumnFamily", true, null); + String [] types = { "INT", "INT", "INT", "DATETIME" }; + String [] vals = { "0", "1", "1", "'2017-03-20'" }; + createTableWithColTypes(types, vals); + runImport(argv); + verifyHBaseCell("AppendTable", "0", "AppendColumnFamily", getColName(2), "1"); + // Run a second time. + argv = getIncrementalArgv(true, "AppendTable", "AppendColumnFamily", true, null, true, false, "DATA_COL1", "2017-03-24 01:01:01.0", null); + vals = new String[] { "1", "2", "3", "'2017-06-15'" }; + insertIntoTable(types, vals); + runImport(argv); + verifyHBaseCell("AppendTable", "1", "AppendColumnFamily", getColName(2), "3"); + } + + @Test + public void testAppendSucceeds() throws IOException { + // Test that we can create a table and then import multiple rows + // validate for append scenario with ID column(DATA_COL3) + String [] argv = getArgv(true, "AppendTable", "AppendColumnFamily", true, null); + String [] types = { "INT", "INT", "INT", "DATETIME" }; + String [] vals = { "0", "1", "1", "'2017-03-20'" }; + createTableWithColTypes(types, vals); + runImport(argv); + verifyHBaseCell("AppendTable", "0", "AppendColumnFamily", getColName(2), "1"); + // Run a second time. + argv = getIncrementalArgv(true, "AppendTable", "AppendColumnFamily", true, null, true, true, "DATA_COL1", null, "DATA_COL3"); + vals = new String[] { "1", "2", "3", "'2017-06-15'" }; + insertIntoTable(types, vals); + runImport(argv); + verifyHBaseCell("AppendTable", "1", "AppendColumnFamily", getColName(2), "3"); + } + + @Test + public void testExitFailure() throws IOException { + String [] types = { "INT", "INT", "INT" }; + String [] vals = { "0", "42", "43" }; + createTableWithColTypes(types, vals); + + String [] argv = getArgv(true, "NoHBaseT", "NoHBaseF", true, null); + try { + HBaseUtil.setAlwaysNoHBaseJarMode(true); + runImport(argv); + } catch (IOException e) { + return; + } finally { + HBaseUtil.setAlwaysNoHBaseJarMode(false); + } + + fail("should have gotten exception"); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseImportTypesTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseImportTypesTest.java b/src/test/org/apache/sqoop/hbase/HBaseImportTypesTest.java new file mode 100644 index 0000000..f3616c3 --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseImportTypesTest.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import java.io.IOException; + +import org.junit.Test; + +/** + * + */ +public class HBaseImportTypesTest extends HBaseTestCase { + + @Test + public void testStrings() throws IOException { + String [] argv = getArgv(true, "stringT", "stringF", true, null); + String [] types = { "INT", "VARCHAR(32)" }; + String [] vals = { "0", "'abc'" }; + createTableWithColTypes(types, vals); + runImport(argv); + verifyHBaseCell("stringT", "0", "stringF", getColName(1), "abc"); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseKerberizedConnectivityTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseKerberizedConnectivityTest.java b/src/test/org/apache/sqoop/hbase/HBaseKerberizedConnectivityTest.java new file mode 100644 index 0000000..73a2247 --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseKerberizedConnectivityTest.java @@ -0,0 +1,33 @@ +package org.apache.sqoop.hbase; + +import org.apache.sqoop.infrastructure.kerberos.MiniKdcInfrastructureRule; +import org.junit.ClassRule; +import org.junit.Test; + +import java.io.IOException; + +public class HBaseKerberizedConnectivityTest extends HBaseTestCase { + + private static final String HBASE_TABLE_NAME = "KerberosTest"; + private static final String HBASE_COLUMN_FAMILY = "TestColumnFamily"; + private static final String TEST_ROW_KEY = "0"; + private static final String TEST_ROW_VALUE = "1"; + private static final String[] COLUMN_TYPES = { "INT", "INT" }; + + @ClassRule + public static MiniKdcInfrastructureRule miniKdcInfrastructure = new MiniKdcInfrastructureRule(); + + public HBaseKerberizedConnectivityTest() { + super(miniKdcInfrastructure); + } + + @Test + public void testSqoopImportWithKerberizedHBaseConnectivitySucceeds() throws IOException { + String[] argv = getArgv(true, HBASE_TABLE_NAME, HBASE_COLUMN_FAMILY, true, null); + createTableWithColTypes(COLUMN_TYPES, new String[] { TEST_ROW_KEY, TEST_ROW_VALUE }); + + runImport(argv); + + verifyHBaseCell(HBASE_TABLE_NAME, TEST_ROW_KEY, HBASE_COLUMN_FAMILY, getColName(1), TEST_ROW_VALUE); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseQueryImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseQueryImportTest.java b/src/test/org/apache/sqoop/hbase/HBaseQueryImportTest.java new file mode 100644 index 0000000..b73afcd --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseQueryImportTest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import java.io.IOException; + +import org.junit.Test; + +import static org.junit.Assert.fail; + +/** + * Test import of free-form query into HBase. + */ +public class HBaseQueryImportTest extends HBaseTestCase { + + @Test + public void testImportFromQuery() throws IOException { + String [] types = { "INT", "INT", "INT" }; + String [] vals = { "0", "42", "43" }; + createTableWithColTypes(types, vals); + + String [] argv = getArgv(true, "queryT", "queryF", true, + "SELECT " + getColName(0) + ", " + getColName(1) + " FROM " + + getTableName() + " WHERE $CONDITIONS"); + runImport(argv); + + // This cell should import correctly. + verifyHBaseCell("queryT", "0", "queryF", getColName(1), "42"); + + // This cell should not be placed in the results.. + verifyHBaseCell("queryT", "0", "queryF", getColName(2), null); + } + + @Test + public void testExitFailure() throws IOException { + String [] types = { "INT", "INT", "INT" }; + String [] vals = { "0", "42", "43" }; + createTableWithColTypes(types, vals); + + String [] argv = getArgv(true, "queryT", "queryF", true, + "SELECT " + getColName(0) + ", " + getColName(1) + " FROM " + + getTableName() + " WHERE $CONDITIONS"); + try { + HBaseUtil.setAlwaysNoHBaseJarMode(true); + runImport(argv); + } catch (Exception e) { + return; + } finally { + HBaseUtil.setAlwaysNoHBaseJarMode(false); + } + fail("should have gotten exception"); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseTestCase.java b/src/test/org/apache/sqoop/hbase/HBaseTestCase.java new file mode 100644 index 0000000..98f8698 --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseTestCase.java @@ -0,0 +1,304 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import static org.apache.hadoop.hbase.HConstants.MASTER_INFO_PORT; +import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT; +import static org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGION_COPROCESSOR_CONF_KEY; +import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.KRB_PRINCIPAL; +import static org.apache.hadoop.hbase.security.HBaseKerberosUtils.MASTER_KRB_PRINCIPAL; +import static org.apache.hadoop.hbase.security.User.HBASE_SECURITY_CONF_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HTTP_POLICY_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.RM_PRINCIPAL; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.security.HBaseKerberosUtils; +import org.apache.hadoop.hbase.security.token.TokenProvider; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.StringUtils; +import org.apache.sqoop.infrastructure.kerberos.KerberosConfigurationProvider; +import org.junit.After; +import org.junit.Before; + +import org.apache.sqoop.testutil.CommonArgs; +import org.apache.sqoop.testutil.HsqldbTestServer; +import org.apache.sqoop.testutil.ImportJobTestCase; + +/** + * Utility methods that facilitate HBase import tests. + */ +public abstract class HBaseTestCase extends ImportJobTestCase { + + public static final Log LOG = LogFactory.getLog( + HBaseTestCase.class.getName()); + private static final String MASTER_INFO_PORT_DISABLE_WEB_UI = "-1"; + private static final String DEFAULT_DFS_HTTPS_ADDRESS = "localhost:0"; + + private final KerberosConfigurationProvider kerberosConfigurationProvider; + private HBaseTestingUtility hbaseTestUtil; + + public HBaseTestCase() { + this(null); + } + + public HBaseTestCase(KerberosConfigurationProvider kerberosConfigurationProvider) { + this.kerberosConfigurationProvider = kerberosConfigurationProvider; + } + + /** + * Create the argv to pass to Sqoop. + * @return the argv as an array of strings. + */ + protected String [] getArgv(boolean includeHadoopFlags, + String hbaseTable, String hbaseColFam, boolean hbaseCreate, + String queryStr) { + + ArrayList<String> args = new ArrayList<String>(); + + if (includeHadoopFlags) { + CommonArgs.addHadoopFlags(args); + String zookeeperPort = hbaseTestUtil.getConfiguration().get(ZOOKEEPER_CLIENT_PORT); + args.add("-D"); + args.add("hbase.zookeeper.property.clientPort=" + zookeeperPort); + args.addAll(getKerberosFlags()); + } + + if (null != queryStr) { + args.add("--query"); + args.add(queryStr); + } else { + args.add("--table"); + args.add(getTableName()); + } + args.add("--split-by"); + args.add(getColName(0)); + args.add("--connect"); + args.add(HsqldbTestServer.getUrl()); + args.add("--num-mappers"); + args.add("1"); + args.add("--column-family"); + args.add(hbaseColFam); + args.add("--hbase-table"); + args.add(hbaseTable); + if (hbaseCreate) { + args.add("--hbase-create-table"); + } + return args.toArray(new String[0]); + } + + /** + * Create the argv to pass to Sqoop as incremental options. + * @return the argv as an array of strings. + */ + protected String [] getIncrementalArgv(boolean includeHadoopFlags, + String hbaseTable, String hbaseColFam, boolean hbaseCreate, + String queryStr, boolean isAppend, boolean appendTimestamp, String checkColumn, String checkValue, String lastModifiedColumn) { + + String[] argsStrArray = getArgv(includeHadoopFlags, hbaseTable, hbaseColFam, hbaseCreate, queryStr); + List<String> args = new ArrayList<String>(Arrays.asList(argsStrArray)); + + if (isAppend) { + args.add("--incremental"); + args.add("append"); + if (!appendTimestamp) { + args.add("--check-column"); + args.add(checkColumn);//"ID"); + } else { + args.add("--check-column"); + args.add(lastModifiedColumn);//LAST_MODIFIED"); + } + } else { + args.add("--incremental"); + args.add("lastmodified"); + args.add("--check-column"); + args.add(checkColumn); + args.add("--last-value"); + args.add(checkValue); + } + return args.toArray(new String[0]); + } + + @Override + @Before + public void setUp() { + try { + hbaseTestUtil = new HBaseTestingUtility(); + // We set the port for the hbase master web UI to -1 because we do not want the info server to run. + hbaseTestUtil.getConfiguration().set(MASTER_INFO_PORT, MASTER_INFO_PORT_DISABLE_WEB_UI); + setupKerberos(); + + hbaseTestUtil.startMiniCluster(); + super.setUp(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + private void setupKerberos() { + if (!isKerberized()){ + return; + } + + String servicePrincipal = kerberosConfigurationProvider.getTestPrincipal() + "@" + kerberosConfigurationProvider.getRealm(); + HBaseKerberosUtils.setPrincipalForTesting(servicePrincipal); + HBaseKerberosUtils.setKeytabFileForTesting(kerberosConfigurationProvider.getKeytabFilePath()); + + Configuration configuration = hbaseTestUtil.getConfiguration(); + HBaseKerberosUtils.setSecuredConfiguration(configuration); + UserGroupInformation.setConfiguration(configuration); + configuration.setStrings(REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); + + setupKerberosForHdfs(servicePrincipal, configuration); + } + + private void setupKerberosForHdfs(String servicePrincipal, Configuration configuration) { + configuration.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, servicePrincipal); + configuration.set(DFS_NAMENODE_KEYTAB_FILE_KEY, kerberosConfigurationProvider.getKeytabFilePath()); + configuration.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, servicePrincipal); + configuration.set(DFS_DATANODE_KEYTAB_FILE_KEY, kerberosConfigurationProvider.getKeytabFilePath()); + configuration.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); + configuration.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, servicePrincipal); + configuration.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTP_ONLY.name()); + configuration.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, DEFAULT_DFS_HTTPS_ADDRESS); + configuration.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, DEFAULT_DFS_HTTPS_ADDRESS); + configuration.setBoolean(IGNORE_SECURE_PORTS_FOR_TESTING_KEY, true); + } + + public void shutdown() throws Exception { + LOG.info("In shutdown() method"); + LOG.info("Shutting down HBase cluster"); + hbaseTestUtil.shutdownMiniCluster(); + hbaseTestUtil = null; + LOG.info("shutdown() method returning."); + } + + @Override + @After + public void tearDown() { + try { + shutdown(); + } catch (Exception e) { + LOG.warn("Error shutting down HBase minicluster: " + + StringUtils.stringifyException(e)); + } + super.tearDown(); + } + + protected void verifyHBaseCell(String tableName, String rowKey, + String colFamily, String colName, String val) throws IOException { + Get get = new Get(Bytes.toBytes(rowKey)); + get.addColumn(Bytes.toBytes(colFamily), Bytes.toBytes(colName)); + try ( + Connection hbaseConnection = createHBaseConnection(); + Table table = getHBaseTable(hbaseConnection, tableName) + ) { + Result r = table.get(get); + byte [] actualVal = r.getValue(Bytes.toBytes(colFamily), + Bytes.toBytes(colName)); + if (null == val) { + assertNull("Got a result when expected null", actualVal); + } else { + assertNotNull("No result, but we expected one", actualVal); + assertEquals(val, Bytes.toString(actualVal)); + } + } + } + + protected int countHBaseTable(String tableName, String colFamily) + throws IOException { + int count = 0; + try ( + Connection hbaseConnection = createHBaseConnection(); + Table table = getHBaseTable(hbaseConnection, tableName) + ) { + ResultScanner scanner = table.getScanner(Bytes.toBytes(colFamily)); + for(Result result = scanner.next(); + result != null; + result = scanner.next()) { + count++; + } + } + return count; + } + + private Connection createHBaseConnection() throws IOException { + return ConnectionFactory.createConnection(new Configuration(hbaseTestUtil.getConfiguration())); + } + + private Table getHBaseTable(Connection connection, String tableName) throws IOException { + return connection.getTable(TableName.valueOf(tableName)); + } + + protected boolean isKerberized() { + return kerberosConfigurationProvider != null; + } + + private String createFlagWithValue(String flag, String value) { + return String.format("%s=%s", flag, value); + } + + private List<String> getKerberosFlags() { + if (!isKerberized()) { + return Collections.emptyList(); + } + List<String> result = new ArrayList<>(); + + String principalForTesting = HBaseKerberosUtils.getPrincipalForTesting(); + result.add("-D"); + result.add(createFlagWithValue(HBASE_SECURITY_CONF_KEY, "kerberos")); + result.add("-D"); + result.add(createFlagWithValue(MASTER_KRB_PRINCIPAL, principalForTesting)); + result.add("-D"); + result.add(createFlagWithValue(KRB_PRINCIPAL, principalForTesting)); + result.add("-D"); + result.add(createFlagWithValue(RM_PRINCIPAL, principalForTesting)); + + return result; + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/HBaseUtilTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/HBaseUtilTest.java b/src/test/org/apache/sqoop/hbase/HBaseUtilTest.java new file mode 100644 index 0000000..c6a808c --- /dev/null +++ b/src/test/org/apache/sqoop/hbase/HBaseUtilTest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hbase; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + + +/** + * This tests to verify that HBase is present (default when running test cases) + * and that when in fake not present mode, the method return false. + */ +public class HBaseUtilTest { + + @Test + public void testHBasePresent() { + assertTrue(HBaseUtil.isHBaseJarPresent()); + } + + @Test + public void testHBaseNotPresent() { + HBaseUtil.setAlwaysNoHBaseJarMode(true); + boolean present = HBaseUtil.isHBaseJarPresent(); + HBaseUtil.setAlwaysNoHBaseJarMode(false); + assertFalse(present); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hbase/TestHBasePutProcessor.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hbase/TestHBasePutProcessor.java b/src/test/org/apache/sqoop/hbase/TestHBasePutProcessor.java index 73b3177..e78a535 100644 --- a/src/test/org/apache/sqoop/hbase/TestHBasePutProcessor.java +++ b/src/test/org/apache/sqoop/hbase/TestHBasePutProcessor.java @@ -17,7 +17,7 @@ */ package org.apache.sqoop.hbase; -import com.cloudera.sqoop.lib.FieldMappable; +import org.apache.sqoop.lib.FieldMappable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.BufferedMutator; import org.apache.hadoop.hbase.client.Connection; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java index 7ff046e..a124dd0 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogExportTest.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; -import com.cloudera.sqoop.SqoopOptions; +import org.apache.sqoop.SqoopOptions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -47,7 +47,7 @@ import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType; import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities; import org.junit.Before; -import com.cloudera.sqoop.testutil.ExportJobTestCase; +import org.apache.sqoop.testutil.ExportJobTestCase; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java index b6741f4..c7e1ea6 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java @@ -42,7 +42,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hive.hcatalog.data.HCatRecord; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.data.schema.HCatSchema; @@ -53,12 +52,12 @@ import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType; import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities; import org.junit.Before; -import com.cloudera.sqoop.Sqoop; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.CommonArgs; -import com.cloudera.sqoop.testutil.ImportJobTestCase; -import com.cloudera.sqoop.tool.ImportTool; -import com.cloudera.sqoop.tool.SqoopTool; +import org.apache.sqoop.Sqoop; +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.testutil.CommonArgs; +import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.tool.ImportTool; +import org.apache.sqoop.tool.SqoopTool; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java index 6fb6486..28b42dc 100644 --- a/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java +++ b/src/test/org/apache/sqoop/hcat/HCatalogTestUtils.java @@ -63,9 +63,9 @@ import org.apache.sqoop.config.ConfigurationConstants; import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities; import org.junit.Assert; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.testutil.BaseSqoopTestCase; -import com.cloudera.sqoop.testutil.CommonArgs; +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.testutil.BaseSqoopTestCase; +import org.apache.sqoop.testutil.CommonArgs; /** * HCatalog common test utilities. http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java index 104effb..ba05cab 100644 --- a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java +++ b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java @@ -20,9 +20,9 @@ package org.apache.sqoop.hcat; import org.junit.Before; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.tool.ExportTool; -import com.cloudera.sqoop.tool.ImportTool; +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.tool.ExportTool; +import org.apache.sqoop.tool.ImportTool; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hive/TestHiveImport.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hive/TestHiveImport.java b/src/test/org/apache/sqoop/hive/TestHiveImport.java new file mode 100644 index 0000000..4e1f249 --- /dev/null +++ b/src/test/org/apache/sqoop/hive/TestHiveImport.java @@ -0,0 +1,743 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.hive; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.sqoop.Sqoop; + +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericRecord; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.sqoop.avro.AvroSchemaMismatchException; +import org.apache.sqoop.mapreduce.ParquetJob; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.SqoopOptions.InvalidOptionsException; +import org.apache.sqoop.testutil.CommonArgs; +import org.apache.sqoop.testutil.ImportJobTestCase; +import org.apache.sqoop.tool.BaseSqoopTool; +import org.apache.sqoop.tool.CodeGenTool; +import org.apache.sqoop.tool.CreateHiveTableTool; +import org.apache.sqoop.tool.ImportTool; +import org.apache.sqoop.tool.SqoopTool; +import org.apache.commons.cli.ParseException; +import org.junit.rules.ExpectedException; +import org.kitesdk.data.Dataset; +import org.kitesdk.data.DatasetReader; +import org.kitesdk.data.Datasets; +import org.kitesdk.data.Formats; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test HiveImport capability after an import to HDFS. + */ + +public class TestHiveImport extends ImportJobTestCase { + + public static final Log LOG = LogFactory.getLog( + TestHiveImport.class.getName()); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Before + public void setUp() { + super.setUp(); + HiveImport.setTestMode(true); + } + + @After + public void tearDown() { + super.tearDown(); + HiveImport.setTestMode(false); + } + + /** + * Sets the expected number of columns in the table being manipulated + * by the test. Under the hood, this sets the expected column names + * to DATA_COLi for 0 <= i < numCols. + * @param numCols the number of columns to be created. + */ + protected void setNumCols(int numCols) { + String [] cols = new String[numCols]; + for (int i = 0; i < numCols; i++) { + cols[i] = "DATA_COL" + i; + } + + setColNames(cols); + } + + protected String[] getTypes() { + String[] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + return types; + } + + /** + * Create the argv to pass to Sqoop. + * @return the argv as an array of strings. + */ + protected String [] getArgv(boolean includeHadoopFlags, String [] moreArgs) { + ArrayList<String> args = new ArrayList<String>(); + + if (includeHadoopFlags) { + CommonArgs.addHadoopFlags(args); + } + + if (null != moreArgs) { + for (String arg: moreArgs) { + args.add(arg); + } + } + + args.add("--table"); + args.add(getTableName()); + args.add("--warehouse-dir"); + args.add(getWarehouseDir()); + args.add("--connect"); + args.add(getConnectString()); + args.add("--hive-import"); + String [] colNames = getColNames(); + if (null != colNames) { + args.add("--split-by"); + args.add(colNames[0]); + } else { + fail("Could not determine column names."); + } + + args.add("--num-mappers"); + args.add("1"); + + for (String a : args) { + LOG.debug("ARG : "+ a); + } + + return args.toArray(new String[0]); + } + + /** + * @return the argv to supply to a create-table only job for Hive imports. + */ + protected String [] getCreateTableArgv(boolean includeHadoopFlags, + String [] moreArgs) { + + ArrayList<String> args = new ArrayList<String>(); + + if (null != moreArgs) { + for (String arg: moreArgs) { + args.add(arg); + } + } + + args.add("--table"); + args.add(getTableName()); + args.add("--connect"); + args.add(getConnectString()); + + return args.toArray(new String[0]); + } + + /** + * @return the argv to supply to a code-gen only job for Hive imports. + */ + protected String [] getCodeGenArgs() { + ArrayList<String> args = new ArrayList<String>(); + + args.add("--table"); + args.add(getTableName()); + args.add("--connect"); + args.add(getConnectString()); + args.add("--hive-import"); + + return args.toArray(new String[0]); + } + + /** + * @return the argv to supply to a ddl-executing-only job for Hive imports. + */ + protected String [] getCreateHiveTableArgs(String [] extraArgs) { + ArrayList<String> args = new ArrayList<String>(); + + args.add("--table"); + args.add(getTableName()); + args.add("--connect"); + args.add(getConnectString()); + + if (null != extraArgs) { + for (String arg : extraArgs) { + args.add(arg); + } + } + + return args.toArray(new String[0]); + } + + private SqoopOptions getSqoopOptions(String [] args, SqoopTool tool) { + SqoopOptions opts = null; + try { + opts = tool.parseArguments(args, null, null, true); + } catch (Exception e) { + fail("Invalid options: " + e.toString()); + } + + return opts; + } + + private void runImportTest(String tableName, String [] types, + String [] values, String verificationScript, String [] args, + SqoopTool tool) throws IOException { + + // create a table and populate it with a row... + createTableWithColTypes(types, values); + + // set up our mock hive shell to compare our generated script + // against the correct expected one. + SqoopOptions options = getSqoopOptions(args, tool); + String hiveHome = options.getHiveHome(); + assertNotNull("hive.home was not set", hiveHome); + String testDataPath = new Path(new Path(hiveHome), + "scripts/" + verificationScript).toString(); + System.setProperty("expected.script", + new File(testDataPath).getAbsolutePath()); + + // verify that we can import it correctly into hive. + runImport(tool, args); + } + + /** Test that we can generate a file containing the DDL and not import. */ + @Test + public void testGenerateOnly() throws IOException { + final String TABLE_NAME = "GenerateOnly"; + setCurTableName(TABLE_NAME); + setNumCols(1); + + // Figure out where our target generated .q file is going to be. + SqoopOptions options = getSqoopOptions(getArgv(false, null), + new ImportTool()); + Path ddlFile = new Path(new Path(options.getCodeOutputDir()), + TABLE_NAME + ".q"); + FileSystem fs = FileSystem.getLocal(new Configuration()); + + // If it's already there, remove it before running the test to ensure + // that it's the current test that generated the file. + if (fs.exists(ddlFile)) { + if (!fs.delete(ddlFile, false)) { + LOG.warn("Could not delete previous ddl file: " + ddlFile); + } + } + + // Run a basic import, but specify that we're just generating definitions. + String [] types = { "INTEGER" }; + String [] vals = { "42" }; + runImportTest(TABLE_NAME, types, vals, null, getCodeGenArgs(), + new CodeGenTool()); + + // Test that the generated definition file exists. + assertTrue("Couldn't find expected ddl file", fs.exists(ddlFile)); + + Path hiveImportPath = new Path(new Path(options.getWarehouseDir()), + TABLE_NAME); + assertFalse("Import actually happened!", fs.exists(hiveImportPath)); + } + + /** Test that strings and ints are handled in the normal fashion. */ + @Test + public void testNormalHiveImport() throws IOException { + final String TABLE_NAME = "NORMAL_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] vals = { "'test'", "42", "'somestring'" }; + runImportTest(TABLE_NAME, types, vals, "normalImport.q", + getArgv(false, null), new ImportTool()); + } + + /** Test that strings and ints are handled in the normal fashion as parquet + * file. */ + @Test + public void testNormalHiveImportAsParquet() throws IOException { + final String TABLE_NAME = "NORMAL_HIVE_IMPORT_AS_PARQUET"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = getTypes(); + String [] vals = { "'test'", "42", "'somestring'" }; + String [] extraArgs = {"--as-parquetfile"}; + + runImportTest(TABLE_NAME, types, vals, "", getArgv(false, extraArgs), + new ImportTool()); + verifyHiveDataset(TABLE_NAME, new Object[][]{{"test", 42, "somestring"}}); + } + + private void verifyHiveDataset(String tableName, Object[][] valsArray) { + String datasetUri = String.format("dataset:hive:default/%s", + tableName.toLowerCase()); + assertTrue(Datasets.exists(datasetUri)); + Dataset dataset = Datasets.load(datasetUri); + assertFalse(dataset.isEmpty()); + + DatasetReader<GenericRecord> reader = dataset.newReader(); + try { + List<String> expectations = new ArrayList<String>(); + if (valsArray != null) { + for (Object[] vals : valsArray) { + expectations.add(Arrays.toString(vals)); + } + } + + while (reader.hasNext() && expectations.size() > 0) { + String actual = Arrays.toString( + convertGenericRecordToArray(reader.next())); + assertTrue("Expect record: " + actual, expectations.remove(actual)); + } + assertFalse(reader.hasNext()); + assertEquals(0, expectations.size()); + } finally { + reader.close(); + } + } + + private static Object[] convertGenericRecordToArray(GenericRecord record) { + Object[] result = new Object[record.getSchema().getFields().size()]; + for (int i = 0; i < result.length; i++) { + result[i] = record.get(i); + } + return result; + } + + /** Test that table is created in hive with no data import. */ + @Test + public void testCreateOnlyHiveImport() throws IOException { + final String TABLE_NAME = "CREATE_ONLY_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] vals = { "'test'", "42", "'somestring'" }; + runImportTest(TABLE_NAME, types, vals, + "createOnlyImport.q", getCreateHiveTableArgs(null), + new CreateHiveTableTool()); + } + + /** + * Test that table is created in hive and replaces the existing table if + * any. + */ + @Test + public void testCreateOverwriteHiveImport() throws IOException { + final String TABLE_NAME = "CREATE_OVERWRITE_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] vals = { "'test'", "42", "'somestring'" }; + String [] extraArgs = {"--hive-overwrite", "--create-hive-table"}; + runImportTest(TABLE_NAME, types, vals, + "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), + new CreateHiveTableTool()); + runImportTest(TABLE_NAME, types, vals, + "createOverwriteImport.q", getCreateHiveTableArgs(extraArgs), + new CreateHiveTableTool()); + } + + /** + * Test that table is created in hive and replaces the existing table if + * any. + */ + @Test + public void testCreateOverwriteHiveImportAsParquet() throws IOException { + final String TABLE_NAME = "CREATE_OVERWRITE_HIVE_IMPORT_AS_PARQUET"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = getTypes(); + String [] vals = { "'test'", "42", "'somestring'" }; + String [] extraArgs = {"--as-parquetfile"}; + ImportTool tool = new ImportTool(); + + runImportTest(TABLE_NAME, types, vals, "", getArgv(false, extraArgs), tool); + verifyHiveDataset(TABLE_NAME, new Object[][]{{"test", 42, "somestring"}}); + + String [] valsToOverwrite = { "'test2'", "24", "'somestring2'" }; + String [] extraArgsForOverwrite = {"--as-parquetfile", "--hive-overwrite"}; + runImportTest(TABLE_NAME, types, valsToOverwrite, "", + getArgv(false, extraArgsForOverwrite), tool); + verifyHiveDataset(TABLE_NAME, new Object[][] {{"test2", 24, "somestring2"}}); + } + + @Test + public void testHiveImportAsParquetWhenTableExistsWithIncompatibleSchema() throws Exception { + final String TABLE_NAME = "HIVE_IMPORT_AS_PARQUET_EXISTING_TABLE"; + setCurTableName(TABLE_NAME); + setNumCols(3); + + String [] types = { "VARCHAR(32)", "INTEGER", "DATE" }; + String [] vals = { "'test'", "42", "'2009-12-31'" }; + String [] extraArgs = {"--as-parquetfile"}; + + createHiveDataSet(TABLE_NAME); + + createTableWithColTypes(types, vals); + + thrown.expect(AvroSchemaMismatchException.class); + thrown.expectMessage(ParquetJob.INCOMPATIBLE_AVRO_SCHEMA_MSG + ParquetJob.HIVE_INCOMPATIBLE_AVRO_SCHEMA_MSG); + + SqoopOptions sqoopOptions = getSqoopOptions(getConf()); + sqoopOptions.setThrowOnError(true); + Sqoop sqoop = new Sqoop(new ImportTool(), getConf(), sqoopOptions); + sqoop.run(getArgv(false, extraArgs)); + + } + + private void createHiveDataSet(String tableName) { + Schema dataSetSchema = SchemaBuilder + .record(tableName) + .fields() + .name(getColName(0)).type().nullable().stringType().noDefault() + .name(getColName(1)).type().nullable().stringType().noDefault() + .name(getColName(2)).type().nullable().stringType().noDefault() + .endRecord(); + String dataSetUri = "dataset:hive:/default/" + tableName; + ParquetJob.createDataset(dataSetSchema, Formats.PARQUET.getDefaultCompressionType(), dataSetUri); + } + + /** + * Test that records are appended to an existing table. + */ + @Test + public void testAppendHiveImportAsParquet() throws IOException { + final String TABLE_NAME = "APPEND_HIVE_IMPORT_AS_PARQUET"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = getTypes(); + String [] vals = { "'test'", "42", "'somestring'" }; + String [] extraArgs = {"--as-parquetfile"}; + String [] args = getArgv(false, extraArgs); + ImportTool tool = new ImportTool(); + + runImportTest(TABLE_NAME, types, vals, "", args, tool); + verifyHiveDataset(TABLE_NAME, new Object[][]{{"test", 42, "somestring"}}); + + String [] valsToAppend = { "'test2'", "4242", "'somestring2'" }; + runImportTest(TABLE_NAME, types, valsToAppend, "", args, tool); + verifyHiveDataset(TABLE_NAME, new Object[][] { + {"test2", 4242, "somestring2"}, {"test", 42, "somestring"}}); + } + + /** + * Test hive create and --as-parquetfile options validation. + */ + @Test + public void testCreateHiveImportAsParquet() throws ParseException, InvalidOptionsException { + final String TABLE_NAME = "CREATE_HIVE_IMPORT_AS_PARQUET"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] extraArgs = {"--as-parquetfile", "--create-hive-table"}; + ImportTool tool = new ImportTool(); + + thrown.expect(InvalidOptionsException.class); + thrown.reportMissingExceptionWithMessage("Expected InvalidOptionsException during Hive table creation with " + + "--as-parquetfile"); + tool.validateOptions(tool.parseArguments(getArgv(false, extraArgs), null, + null, true)); + } + + + /** Test that dates are coerced properly to strings. */ + @Test + public void testDate() throws IOException { + final String TABLE_NAME = "DATE_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "VARCHAR(32)", "DATE" }; + String [] vals = { "'test'", "'2009-05-12'" }; + runImportTest(TABLE_NAME, types, vals, "dateImport.q", + getArgv(false, null), new ImportTool()); + } + + /** Test that NUMERICs are coerced to doubles. */ + @Test + public void testNumeric() throws IOException { + final String TABLE_NAME = "NUMERIC_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "NUMERIC", "CHAR(64)" }; + String [] vals = { "3.14159", "'foo'" }; + runImportTest(TABLE_NAME, types, vals, "numericImport.q", + getArgv(false, null), new ImportTool()); + } + + /** Test that DECIMALS using --map-column-hive option maps can run without issues. */ + @Test + public void testDecimalMapColumnHive() throws IOException { + final String TABLE_NAME = "DECIMAL_MAP_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "NUMERIC", "CHAR(64)" }; + String [] vals = { "12343.14159", "'foo'" }; + + ArrayList<String> args = new ArrayList<String>(); + args.add("--map-column-hive"); + args.add(BASE_COL_NAME + "0=DECIMAL(10,10)"); + + runImportTest(TABLE_NAME, types, vals, "decimalMapImport.q", + getArgv(false, args.toArray(new String[args.size()])), new ImportTool()); + } + + /** If bin/hive returns an error exit status, we should get an IOException. */ + @Test + public void testHiveExitFails() throws IOException { + // The expected script is different than the one which would be generated + // by this, so we expect an IOException out. + final String TABLE_NAME = "FAILING_HIVE_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(2); + String [] types = { "NUMERIC", "CHAR(64)" }; + String [] vals = { "3.14159", "'foo'" }; + + thrown.expect(IOException.class); + thrown.reportMissingExceptionWithMessage("Expected IOException on erroneous Hive exit status"); + runImportTest(TABLE_NAME, types, vals, "failingImport.q", + getArgv(false, null), new ImportTool()); + } + + /** Test that we can set delimiters how we want them. */ + @Test + public void testCustomDelimiters() throws IOException { + final String TABLE_NAME = "CUSTOM_DELIM_IMPORT"; + setCurTableName(TABLE_NAME); + setNumCols(3); + String [] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)" }; + String [] vals = { "'test'", "42", "'somestring'" }; + String [] extraArgs = { + "--fields-terminated-by", ",", + "--lines-terminated-by", "|", + }; + runImportTest(TABLE_NAME, types, vals, "customDelimImport.q", + getArgv(false, extraArgs), new ImportTool()); + } + + /** + * Test hive import with row that has new line in it. + */ + @Test + public void testFieldWithHiveDelims() throws IOException, + InterruptedException { + final String TABLE_NAME = "FIELD_WITH_NL_HIVE_IMPORT"; + + LOG.info("Doing import of single row into FIELD_WITH_NL_HIVE_IMPORT table"); + setCurTableName(TABLE_NAME); + setNumCols(3); + String[] types = getTypes(); + String[] vals = { "'test with \n new lines \n'", "42", + "'oh no " + '\01' + " field delims " + '\01' + "'", }; + String[] moreArgs = { "--"+ BaseSqoopTool.HIVE_DROP_DELIMS_ARG }; + + runImportTest(TABLE_NAME, types, vals, "fieldWithNewlineImport.q", + getArgv(false, moreArgs), new ImportTool()); + + LOG.info("Validating data in single row is present in: " + + "FIELD_WITH_NL_HIVE_IMPORT table"); + + // Ideally, we would actually invoke hive code to verify that record with + // record and field delimiters have values replaced and that we have the + // proper number of hive records. Unfortunately, this is a non-trivial task, + // and better dealt with at an integration test level + // + // Instead, this assumes the path of the generated table and just validate + // map job output. + + // Get and read the raw output file + String whDir = getWarehouseDir(); + File p = new File(new File(whDir, TABLE_NAME), "part-m-00000"); + File f = new File(p.toString()); + FileReader fr = new FileReader(f); + BufferedReader br = new BufferedReader(fr); + try { + // verify the output + assertEquals(br.readLine(), "test with new lines " + '\01' + "42" + + '\01' + "oh no field delims "); + assertEquals(br.readLine(), null); // should only be one line + } catch (IOException ioe) { + fail("Unable to read files generated from hive"); + } finally { + br.close(); + } + } + + /** + * Test hive import with row that has new line in it. + */ + @Test + public void testFieldWithHiveDelimsReplacement() throws IOException, + InterruptedException { + final String TABLE_NAME = "FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT"; + + LOG.info("Doing import of single row into " + + "FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT table"); + setCurTableName(TABLE_NAME); + setNumCols(3); + String[] types = getTypes(); + String[] vals = { "'test with\nnew lines\n'", "42", + "'oh no " + '\01' + " field delims " + '\01' + "'", }; + String[] moreArgs = { "--"+BaseSqoopTool.HIVE_DELIMS_REPLACEMENT_ARG, " "}; + + runImportTest(TABLE_NAME, types, vals, + "fieldWithNewlineReplacementImport.q", getArgv(false, moreArgs), + new ImportTool()); + + LOG.info("Validating data in single row is present in: " + + "FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT table"); + + // Ideally, we would actually invoke hive code to verify that record with + // record and field delimiters have values replaced and that we have the + // proper number of hive records. Unfortunately, this is a non-trivial task, + // and better dealt with at an integration test level + // + // Instead, this assumes the path of the generated table and just validate + // map job output. + + // Get and read the raw output file + String whDir = getWarehouseDir(); + File p = new File(new File(whDir, TABLE_NAME), "part-m-00000"); + File f = new File(p.toString()); + FileReader fr = new FileReader(f); + BufferedReader br = new BufferedReader(fr); + try { + // verify the output + assertEquals(br.readLine(), "test with new lines " + '\01' + "42" + + '\01' + "oh no field delims "); + assertEquals(br.readLine(), null); // should only be one line + } catch (IOException ioe) { + fail("Unable to read files generated from hive"); + } finally { + br.close(); + } + } + + /** + * Test hive drop and replace option validation. + */ + @Test + public void testHiveDropAndReplaceOptionValidation() throws ParseException, InvalidOptionsException { + LOG.info("Testing conflicting Hive delimiter drop/replace options"); + + setNumCols(3); + String[] moreArgs = { "--"+BaseSqoopTool.HIVE_DELIMS_REPLACEMENT_ARG, " ", + "--"+BaseSqoopTool.HIVE_DROP_DELIMS_ARG, }; + + ImportTool tool = new ImportTool(); + + thrown.expect(InvalidOptionsException.class); + thrown.reportMissingExceptionWithMessage("Expected InvalidOptionsException with conflicting Hive delimiter " + + "drop/replace options"); + tool.validateOptions(tool.parseArguments(getArgv(false, moreArgs), null, + null, true)); + } + + /** + * Test hive import with row that has new line in it. + */ + @Test + public void testImportHiveWithPartitions() throws IOException, + InterruptedException { + final String TABLE_NAME = "PARTITION_HIVE_IMPORT"; + + LOG.info("Doing import of single row into PARTITION_HIVE_IMPORT table"); + setCurTableName(TABLE_NAME); + setNumCols(3); + String[] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)", }; + String[] vals = { "'whoop'", "42", "'I am a row in a partition'", }; + String[] moreArgs = { "--" + BaseSqoopTool.HIVE_PARTITION_KEY_ARG, "ds", + "--" + BaseSqoopTool.HIVE_PARTITION_VALUE_ARG, "20110413", }; + + runImportTest(TABLE_NAME, types, vals, "partitionImport.q", + getArgv(false, moreArgs), new ImportTool()); + } + + /** + * If partition key is set to one of importing columns, we should get an + * IOException. + * */ + @Test + public void testImportWithBadPartitionKey() throws IOException { + final String TABLE_NAME = "FAILING_PARTITION_HIVE_IMPORT"; + + LOG.info("Doing import of single row into " + TABLE_NAME + " table"); + setCurTableName(TABLE_NAME); + setNumCols(3); + String[] types = { "VARCHAR(32)", "INTEGER", "CHAR(64)", }; + String[] vals = { "'key'", "42", "'I am a row in a partition'", }; + + String partitionKey = getColNames()[0]; + + // Specify 1st column as partition key and import every column of the + // table by default (i.e. no --columns option). + String[] moreArgs1 = { + "--" + BaseSqoopTool.HIVE_PARTITION_KEY_ARG, + partitionKey, + }; + + // Specify 1st column as both partition key and importing column. + String[] moreArgs2 = { + "--" + BaseSqoopTool.HIVE_PARTITION_KEY_ARG, + partitionKey, + "--" + BaseSqoopTool.COLUMNS_ARG, + partitionKey, + }; + + // Test hive-import with the 1st args. + thrown.expect(IOException.class); + thrown.reportMissingExceptionWithMessage("Expected IOException during Hive import with partition key " + + "as importing column"); + runImportTest(TABLE_NAME, types, vals, "partitionImport.q", + getArgv(false, moreArgs1), new ImportTool()); + + // Test hive-import with the 2nd args. + thrown.expect(IOException.class); + thrown.reportMissingExceptionWithMessage("Expected IOException during Hive import with partition key " + + "as importing column"); + runImportTest(TABLE_NAME, types, vals, "partitionImport.q", + getArgv(false, moreArgs2), new ImportTool()); + + // Test create-hive-table with the 1st args. + thrown.expect(IOException.class); + thrown.reportMissingExceptionWithMessage("Expected IOException during Hive table creation with partition key " + + "as importing column"); + runImportTest(TABLE_NAME, types, vals, "partitionImport.q", + getCreateTableArgv(false, moreArgs1), new CreateHiveTableTool()); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hive/TestTableDefWriter.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hive/TestTableDefWriter.java b/src/test/org/apache/sqoop/hive/TestTableDefWriter.java index 035b0e1..496b5ad 100644 --- a/src/test/org/apache/sqoop/hive/TestTableDefWriter.java +++ b/src/test/org/apache/sqoop/hive/TestTableDefWriter.java @@ -15,112 +15,275 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.sqoop.hive; -import static org.mockito.Mockito.*; +package org.apache.sqoop.hive; -import com.cloudera.sqoop.manager.ConnManager; -import com.cloudera.sqoop.SqoopOptions; +import java.util.Map; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.junit.BeforeClass; +import org.apache.hadoop.conf.Configuration; +import org.apache.sqoop.util.SqlTypeMap; + +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.tool.ImportTool; +import org.apache.sqoop.testutil.HsqldbTestServer; + +import org.junit.Rule; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.rules.ExpectedException; -import java.sql.*; -import java.util.HashMap; -import java.io.IOException; +import java.sql.Types; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + + +/** + * Test Hive DDL statement generation. + */ public class TestTableDefWriter { - static String inputTableName = "genres"; - static String outputTableName = "genres"; - static String testTargetDir = "/tmp/testDB/genre"; - static String hdfsTableDir = "/data/movielens/genre"; - static String testDbUri = "jdbc:postgresql://localhost/movielens"; - static ConnManager manager; - static SqoopOptions options; + public static final Log LOG = LogFactory.getLog( TestTableDefWriter.class.getName()); - TableDefWriter tableDefWriter; - - @BeforeClass - public static void setup() { - // create mock - HashMap<String, Integer> map = new HashMap<String, Integer>(); - map.put("id", Types.TINYINT); - map.put("name", Types.VARCHAR); - manager = Mockito.mock(ConnManager.class); - when(manager.getColumnNames(inputTableName)).thenReturn(new String[] { "id", "name" }); - when(manager.getColumnTypes(inputTableName)).thenReturn(map); - options = new SqoopOptions(testDbUri, inputTableName); - options.setTargetDir(testTargetDir); - options.setHiveExternalTableDir(hdfsTableDir); - String[] cols = new String[] { "id", "name" }; - options.setColumns(cols); - options.setMapColumnHive("id=TINYINT,name=STRING"); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + // Test getHiveOctalCharCode and expect an IllegalArgumentException. + private void expectExceptionInCharCode(int charCode) { + thrown.expect(IllegalArgumentException.class); + thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException with out-of-range Hive delimiter"); + TableDefWriter.getHiveOctalCharCode(charCode); } @Test - public void testGenerateExternalTableStatement() throws IOException, SQLException { - // need to set this as the other unit test functions may override it for their own test. - options.setHiveExternalTableDir(hdfsTableDir); - tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, - options.getConf(), false); - String stmt = tableDefWriter.getCreateTableStmt(); - Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); - LOG.debug("External table dir: "+options.getHiveExternalTableDir()); - assert (isHiveExternalTableSet && stmt.contains("CREATE EXTERNAL TABLE ") && stmt.contains("LOCATION '" + hdfsTableDir)); + public void testHiveOctalCharCode() { + assertEquals("\\000", TableDefWriter.getHiveOctalCharCode(0)); + assertEquals("\\001", TableDefWriter.getHiveOctalCharCode(1)); + assertEquals("\\012", TableDefWriter.getHiveOctalCharCode((int) '\n')); + assertEquals("\\177", TableDefWriter.getHiveOctalCharCode(0177)); + + expectExceptionInCharCode(4096); + expectExceptionInCharCode(0200); + expectExceptionInCharCode(254); } @Test - public void testGenerateTableStatement() throws IOException, SQLException { - // need to set this as the other unit test functions may override it for their own test. - options.setHiveExternalTableDir(null); - tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, - options.getConf(), false); - String stmt = tableDefWriter.getCreateTableStmt(); - Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); - LOG.debug("External table dir: "+options.getHiveExternalTableDir()); - assert (!isHiveExternalTableSet && stmt.contains("CREATE TABLE ")); + public void testDifferentTableNames() throws Exception { + Configuration conf = new Configuration(); + SqoopOptions options = new SqoopOptions(); + TableDefWriter writer = new TableDefWriter(options, null, + "inputTable", "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + String loadData = writer.getLoadDataStmt(); + + LOG.debug("Create table stmt: " + createTable); + LOG.debug("Load data stmt: " + loadData); + + // Assert that the statements generated have the form we expect. + assertTrue(createTable.indexOf( + "CREATE TABLE IF NOT EXISTS `outputTable`") != -1); + assertTrue(loadData.indexOf("INTO TABLE `outputTable`") != -1); + assertTrue(loadData.indexOf("/inputTable'") != -1); } @Test - public void testGenerateExternalTableIfExistsStatement() throws IOException, SQLException { - options.setFailIfHiveTableExists(false); - // need to set this as the other unit test functions may override it for their own test. - options.setHiveExternalTableDir(hdfsTableDir); - tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, - options.getConf(), false); - String stmt = tableDefWriter.getCreateTableStmt(); - Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); - LOG.debug("External table dir: "+options.getHiveExternalTableDir()); - assert (isHiveExternalTableSet && stmt.contains("CREATE EXTERNAL TABLE IF NOT EXISTS") && stmt.contains("LOCATION '" - + hdfsTableDir)); + public void testDifferentTargetDirs() throws Exception { + String targetDir = "targetDir"; + String inputTable = "inputTable"; + String outputTable = "outputTable"; + + Configuration conf = new Configuration(); + SqoopOptions options = new SqoopOptions(); + // Specify a different target dir from input table name + options.setTargetDir(targetDir); + TableDefWriter writer = new TableDefWriter(options, null, + inputTable, outputTable, conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + String loadData = writer.getLoadDataStmt(); + + LOG.debug("Create table stmt: " + createTable); + LOG.debug("Load data stmt: " + loadData); + + // Assert that the statements generated have the form we expect. + assertTrue(createTable.indexOf( + "CREATE TABLE IF NOT EXISTS `" + outputTable + "`") != -1); + assertTrue(loadData.indexOf("INTO TABLE `" + outputTable + "`") != -1); + assertTrue(loadData.indexOf("/" + targetDir + "'") != -1); } @Test - public void testGenerateTableIfExistsStatement() throws IOException, SQLException { - // need to set this as the other unit test functions may override it for their own test. - options.setHiveExternalTableDir(null); - tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, - options.getConf(), false); - String stmt = tableDefWriter.getCreateTableStmt(); - Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); - LOG.debug("External table dir: "+options.getHiveExternalTableDir()); - assert (!isHiveExternalTableSet && stmt.contains("CREATE TABLE IF NOT EXISTS")); + public void testPartitions() throws Exception { + String[] args = { + "--hive-partition-key", "ds", + "--hive-partition-value", "20110413", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, "inputTable", "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + String loadData = writer.getLoadDataStmt(); + + assertNotNull(createTable); + assertNotNull(loadData); + assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) " + + "PARTITIONED BY (ds STRING) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' " + + "LINES TERMINATED BY '\\012' STORED AS TEXTFILE", createTable); + assertTrue(loadData.endsWith(" PARTITION (ds='20110413')")); } @Test - public void testGenerateExternalTableLoadStatement() throws IOException, SQLException { - // need to set this as the other unit test functions may override it for their own test. - options.setHiveExternalTableDir(hdfsTableDir); - tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, - options.getConf(), false); - String stmt = tableDefWriter.getLoadDataStmt(); - Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); - LOG.debug("External table dir: "+options.getHiveExternalTableDir()); - assert (isHiveExternalTableSet && stmt.contains("LOAD DATA INPATH ") && stmt.contains(testTargetDir)); + public void testLzoSplitting() throws Exception { + String[] args = { + "--compress", + "--compression-codec", "lzop", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, "inputTable", "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + String loadData = writer.getLoadDataStmt(); + + assertNotNull(createTable); + assertNotNull(loadData); + assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) " + + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' " + + "LINES TERMINATED BY '\\012' STORED AS " + + "INPUTFORMAT 'com.hadoop.mapred.DeprecatedLzoTextInputFormat' " + + "OUTPUTFORMAT " + + "'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'", + createTable); } + + @Test + public void testUserMappingNoDecimal() throws Exception { + String[] args = { + "--map-column-hive", "id=STRING,value=INTEGER", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + colTypes.put("id", Types.INTEGER); + colTypes.put("value", Types.VARCHAR); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + + assertNotNull(createTable); + + assertTrue(createTable.contains("`id` STRING")); + assertTrue(createTable.contains("`value` INTEGER")); + + assertFalse(createTable.contains("`id` INTEGER")); + assertFalse(createTable.contains("`value` STRING")); + } + + @Test + public void testUserMappingWithDecimal() throws Exception { + String[] args = { + "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," + + "value3=DECIMAL(4,5),value4=VARCHAR(255)", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + colTypes.put("id", Types.INTEGER); + colTypes.put("value1", Types.VARCHAR); + colTypes.put("value2", Types.DOUBLE); + colTypes.put("value3", Types.FLOAT); + colTypes.put("value4", Types.CHAR); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + + assertNotNull(createTable); + + assertTrue(createTable.contains("`id` STRING")); + assertTrue(createTable.contains("`value1` INTEGER")); + assertTrue(createTable.contains("`value2` DECIMAL(13,5)")); + assertTrue(createTable.contains("`value3` DECIMAL(4,5)")); + assertTrue(createTable.contains("`value4` VARCHAR(255)")); + + assertFalse(createTable.contains("`id` INTEGER")); + assertFalse(createTable.contains("`value1` STRING")); + assertFalse(createTable.contains("`value2` DOUBLE")); + assertFalse(createTable.contains("`value3` FLOAT")); + assertFalse(createTable.contains("`value4` CHAR")); + } + + @Test + public void testUserMappingFailWhenCantBeApplied() throws Exception { + String[] args = { + "--map-column-hive", "id=STRING,value=INTEGER", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + colTypes.put("id", Types.INTEGER); + writer.setColumnTypes(colTypes); + + thrown.expect(IllegalArgumentException.class); + thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException on non applied Hive type mapping"); + String createTable = writer.getCreateTableStmt(); + } + + @Test + public void testHiveDatabase() throws Exception { + String[] args = { + "--hive-database", "db", + }; + Configuration conf = new Configuration(); + SqoopOptions options = + new ImportTool().parseArguments(args, null, null, false); + TableDefWriter writer = new TableDefWriter(options, + null, HsqldbTestServer.getTableName(), "outputTable", conf, false); + + Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>(); + writer.setColumnTypes(colTypes); + + String createTable = writer.getCreateTableStmt(); + assertNotNull(createTable); + assertTrue(createTable.contains("`db`.`outputTable`")); + + String loadStmt = writer.getLoadDataStmt(); + assertNotNull(loadStmt); + assertTrue(createTable.contains("`db`.`outputTable`")); + } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/hive/TestTableDefWriterForExternalTable.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/hive/TestTableDefWriterForExternalTable.java b/src/test/org/apache/sqoop/hive/TestTableDefWriterForExternalTable.java new file mode 100644 index 0000000..f1768ee --- /dev/null +++ b/src/test/org/apache/sqoop/hive/TestTableDefWriterForExternalTable.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.hive; + +import static org.mockito.Mockito.*; + +import org.apache.sqoop.manager.ConnManager; +import org.apache.sqoop.SqoopOptions; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +import java.sql.*; +import java.util.HashMap; +import java.io.IOException; + +public class TestTableDefWriterForExternalTable { + static String inputTableName = "genres"; + static String outputTableName = "genres"; + static String testTargetDir = "/tmp/testDB/genre"; + static String hdfsTableDir = "/data/movielens/genre"; + static String testDbUri = "jdbc:postgresql://localhost/movielens"; + static ConnManager manager; + static SqoopOptions options; + public static final Log LOG = LogFactory.getLog( + TestTableDefWriterForExternalTable.class.getName()); + TableDefWriter tableDefWriter; + + @BeforeClass + public static void setup() { + // create mock + HashMap<String, Integer> map = new HashMap<String, Integer>(); + map.put("id", Types.TINYINT); + map.put("name", Types.VARCHAR); + manager = Mockito.mock(ConnManager.class); + when(manager.getColumnNames(inputTableName)).thenReturn(new String[] { "id", "name" }); + when(manager.getColumnTypes(inputTableName)).thenReturn(map); + options = new SqoopOptions(testDbUri, inputTableName); + options.setTargetDir(testTargetDir); + options.setHiveExternalTableDir(hdfsTableDir); + String[] cols = new String[] { "id", "name" }; + options.setColumns(cols); + options.setMapColumnHive("id=TINYINT,name=STRING"); + } + + @Test + public void testGenerateExternalTableStatement() throws IOException, SQLException { + // need to set this as the other unit test functions may override it for their own test. + options.setHiveExternalTableDir(hdfsTableDir); + tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, + options.getConf(), false); + String stmt = tableDefWriter.getCreateTableStmt(); + Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); + LOG.debug("External table dir: "+options.getHiveExternalTableDir()); + assert (isHiveExternalTableSet && stmt.contains("CREATE EXTERNAL TABLE ") && stmt.contains("LOCATION '" + hdfsTableDir)); + } + + @Test + public void testGenerateTableStatement() throws IOException, SQLException { + // need to set this as the other unit test functions may override it for their own test. + options.setHiveExternalTableDir(null); + tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, + options.getConf(), false); + String stmt = tableDefWriter.getCreateTableStmt(); + Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); + LOG.debug("External table dir: "+options.getHiveExternalTableDir()); + assert (!isHiveExternalTableSet && stmt.contains("CREATE TABLE ")); + } + + @Test + public void testGenerateExternalTableIfExistsStatement() throws IOException, SQLException { + options.setFailIfHiveTableExists(false); + // need to set this as the other unit test functions may override it for their own test. + options.setHiveExternalTableDir(hdfsTableDir); + tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, + options.getConf(), false); + String stmt = tableDefWriter.getCreateTableStmt(); + Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); + LOG.debug("External table dir: "+options.getHiveExternalTableDir()); + assert (isHiveExternalTableSet && stmt.contains("CREATE EXTERNAL TABLE IF NOT EXISTS") && stmt.contains("LOCATION '" + + hdfsTableDir)); + } + + @Test + public void testGenerateTableIfExistsStatement() throws IOException, SQLException { + // need to set this as the other unit test functions may override it for their own test. + options.setHiveExternalTableDir(null); + tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, + options.getConf(), false); + String stmt = tableDefWriter.getCreateTableStmt(); + Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); + LOG.debug("External table dir: "+options.getHiveExternalTableDir()); + assert (!isHiveExternalTableSet && stmt.contains("CREATE TABLE IF NOT EXISTS")); + } + + @Test + public void testGenerateExternalTableLoadStatement() throws IOException, SQLException { + // need to set this as the other unit test functions may override it for their own test. + options.setHiveExternalTableDir(hdfsTableDir); + tableDefWriter = new TableDefWriter(options, manager, inputTableName, outputTableName, + options.getConf(), false); + String stmt = tableDefWriter.getLoadDataStmt(); + Boolean isHiveExternalTableSet = !StringUtils.isBlank(options.getHiveExternalTableDir()); + LOG.debug("External table dir: "+options.getHiveExternalTableDir()); + assert (isHiveExternalTableSet && stmt.contains("LOAD DATA INPATH ") && stmt.contains(testTargetDir)); + } +}
