Author: blee
Date: Tue Apr 3 21:47:15 2012
New Revision: 1309163
URL: http://svn.apache.org/viewvc?rev=1309163&view=rev
Log:
SQOOP-473 Sqoop import-all-tables fails with MySQL connector
Added:
sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
(with props)
Modified:
sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java
Modified: sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java Tue Apr 3
21:47:15 2012
@@ -165,7 +165,7 @@ public class TableDefWriter {
Integer colType = columnTypes.get(col);
String hiveColType = userMapping.getProperty(col);
if (hiveColType == null) {
- hiveColType = connManager.toHiveType(col, colType);
+ hiveColType = connManager.toHiveType(inputTableName, col, colType);
}
if (null == hiveColType) {
throw new IOException("Hive does not support the SQL type for column "
Modified: sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java Tue Apr 3
21:47:15 2012
@@ -203,34 +203,37 @@ public abstract class ConnManager {
/**
* Return java type for SQL type.
+ * @param tableName table name
* @param columnName column name
* @param sqlType sql type
* @return java type
*/
- public String toJavaType(String columnName, int sqlType) {
- // ignore column name by default.
+ public String toJavaType(String tableName, String columnName, int sqlType) {
+ // ignore table name and column name by default.
return toJavaType(sqlType);
}
- /**
- * Return hive type for SQL type.
- * @param columnName column name
- * @param sqlType sql type
- * @return hive type
- */
- public String toHiveType(String columnName, int sqlType) {
- // ignore column name by default.
+ /**
+ * Return hive type for SQL type.
+ * @param tableName table name
+ * @param columnName column name
+ * @param sqlType sql type
+ * @return hive type
+ */
+ public String toHiveType(String tableName, String columnName, int sqlType) {
+ // ignore table name and column name by default.
return toHiveType(sqlType);
}
/**
* Return avro type for SQL type.
+ * @param tableName table name
* @param columnName column name
- * @param sqlType sql type
- * @return avro type
+ * @param sqlType sql type
+ * @return avro type
*/
- public Type toAvroType(String columnName, int sqlType) {
- // ignore column name by default.
+ public Type toAvroType(String tableName, String columnName, int sqlType) {
+ // ignore table name and column name by default.
return toAvroType(sqlType);
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java Tue Apr 3
21:47:15 2012
@@ -232,45 +232,42 @@ public class MySQLManager
private static final int YEAR_TYPE_OVERWRITE = Types.SMALLINT;
@Override
- public String toJavaType(String columnName, int sqlType) {
+ public String toJavaType(String tableName, String columnName, int sqlType) {
if (colTypeNames == null) {
- colTypeNames = getColumnTypeNames(options.getTableName(),
- options.getSqlQuery());
+ colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
sqlType = YEAR_TYPE_OVERWRITE;
}
- return super.toJavaType(columnName, sqlType);
+ return super.toJavaType(tableName, columnName, sqlType);
}
@Override
- public String toHiveType(String columnName, int sqlType) {
+ public String toHiveType(String tableName, String columnName, int sqlType) {
if (colTypeNames == null) {
- colTypeNames = getColumnTypeNames(options.getTableName(),
- options.getSqlQuery());
+ colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
sqlType = YEAR_TYPE_OVERWRITE;
}
- return super.toHiveType(columnName, sqlType);
+ return super.toHiveType(tableName, columnName, sqlType);
}
@Override
- public Type toAvroType(String columnName, int sqlType) {
+ public Type toAvroType(String tableName, String columnName, int sqlType) {
if (colTypeNames == null) {
- colTypeNames = getColumnTypeNames(options.getTableName(),
- options.getSqlQuery());
+ colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
}
if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
sqlType = YEAR_TYPE_OVERWRITE;
}
- return super.toAvroType(columnName, sqlType);
+ return super.toAvroType(tableName, columnName, sqlType);
}
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java Tue Apr 3
21:47:15 2012
@@ -471,11 +471,14 @@ public class OracleManager
/**
* Resolve a database-specific type to the Java type that should contain it.
- * @param sqlType
+ * @param tableName table name
+ * @param columnName column name
+ * @param sqlType sql data type
* @return the name of a Java type to hold the sql datatype, or null if none.
*/
- public String toJavaType(String columnName, int sqlType) {
- String defaultJavaType = super.toJavaType(columnName, sqlType);
+ @Override
+ public String toJavaType(String tableName, String columnName, int sqlType) {
+ String defaultJavaType = super.toJavaType(tableName, columnName, sqlType);
return (defaultJavaType == null) ? dbToJavaType(sqlType) : defaultJavaType;
}
@@ -508,11 +511,13 @@ public class OracleManager
/**
* Attempt to map sql type to hive type.
+ * @param tableName table name
+ * @param columnName column name
* @param sqlType sql data type
* @return hive data type
*/
- public String toHiveType(String columnName, int sqlType) {
- String defaultHiveType = super.toHiveType(columnName, sqlType);
+ public String toHiveType(String tableName, String columnName, int sqlType) {
+ String defaultHiveType = super.toHiveType(tableName, columnName, sqlType);
return (defaultHiveType == null) ? dbToHiveType(sqlType) : defaultHiveType;
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java Tue Apr
3 21:47:15 2012
@@ -74,7 +74,7 @@ public class JdbcExportJob extends Expor
for (Map.Entry<String, Integer> e : columnTypeInts.entrySet()) {
Text columnName = new Text(e.getKey());
Text columnText = new Text(
- connManager.toJavaType(e.getKey(), e.getValue()));
+ connManager.toJavaType(tableName, e.getKey(), e.getValue()));
columnTypes.put(columnName, columnText);
}
DefaultStringifier.store(job.getConfiguration(), columnTypes,
Modified: sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
(original)
+++ sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java Tue Apr
3 21:47:15 2012
@@ -121,6 +121,6 @@ public class AvroSchemaGenerator {
throw new IllegalArgumentException("Cannot convert to AVRO type " +
type);
}
- return connManager.toAvroType(columnName, sqlType);
+ return connManager.toAvroType(tableName, columnName, sqlType);
}
}
Modified: sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java Tue Apr 3
21:47:15 2012
@@ -250,7 +250,7 @@ public class ClassWriter {
return type;
}
- return connManager.toJavaType(columnName, sqlType);
+ return connManager.toJavaType(tableName, columnName, sqlType);
}
/**
Modified: sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java (original)
+++ sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java Tue Apr 3
21:47:15 2012
@@ -30,7 +30,6 @@ import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.HsqldbTestServer;
import com.cloudera.sqoop.testutil.ImportJobTestCase;
import com.cloudera.sqoop.tool.ImportAllTablesTool;
@@ -53,7 +52,7 @@ public class TestAllTables extends Impor
args.add("--warehouse-dir");
args.add(getWarehouseDir());
args.add("--connect");
- args.add(HsqldbTestServer.getUrl());
+ args.add(getConnectString());
args.add("--num-mappers");
args.add("1");
args.add("--escaped-by");
@@ -73,11 +72,13 @@ public class TestAllTables extends Impor
// start the server
super.setUp();
- // throw away TWOINTTABLE and things we don't care about.
- try {
- this.getTestServer().dropExistingSchema();
- } catch (SQLException sqlE) {
- fail(sqlE.toString());
+ if (useHsqldbTestServer()) {
+ // throw away TWOINTTABLE and things we don't care about.
+ try {
+ this.getTestServer().dropExistingSchema();
+ } catch (SQLException sqlE) {
+ fail(sqlE.toString());
+ }
}
this.tableNames = new ArrayList<String>();
Added: sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
URL:
http://svn.apache.org/viewvc/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java?rev=1309163&view=auto
==============================================================================
--- sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
(added)
+++ sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java Tue
Apr 3 21:47:15 2012
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop.manager;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.TestAllTables;
+
+/**
+ * Test the --all-tables functionality with MySQL.
+ */
+public class MySQLAllTablesTest extends TestAllTables {
+
+ @Override
+ protected boolean useHsqldbTestServer() {
+ return false;
+ }
+
+ @Override
+ protected String getConnectString() {
+ return MySQLTestUtils.CONNECT_STRING;
+ }
+
+ @Override
+ protected SqoopOptions getSqoopOptions(Configuration conf) {
+ SqoopOptions opts = new SqoopOptions(conf);
+ opts.setUsername(MySQLTestUtils.getCurrentUser());
+ return opts;
+ }
+
+ @Override
+ protected void dropTableIfExists(String table) throws SQLException {
+ Connection conn = getManager().getConnection();
+ PreparedStatement statement = conn.prepareStatement(
+ "DROP TABLE IF EXISTS " + table,
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ try {
+ statement.executeUpdate();
+ conn.commit();
+ } finally {
+ statement.close();
+ }
+ }
+}
Propchange:
sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
