Repository: sqoop Updated Branches: refs/heads/trunk 9aac957b9 -> 1e2bc6e72
SQOOP-1358: Add wallet support for Oracle High performance connector (Venkat Ranganathan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/1e2bc6e7 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/1e2bc6e7 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/1e2bc6e7 Branch: refs/heads/trunk Commit: 1e2bc6e72b4f9393b6816dedb29e4544fbaeb254 Parents: 9aac957 Author: Jarek Jarcec Cecho <[email protected]> Authored: Sat Jul 19 13:18:49 2014 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Sat Jul 19 13:18:49 2014 -0700 ---------------------------------------------------------------------- src/docs/user/connectors.txt | 9 ++- .../org/apache/sqoop/manager/OracleManager.java | 5 +- .../sqoop/manager/oracle/OraOopConnManager.java | 7 +- .../sqoop/manager/oracle/OraOopConstants.java | 3 + .../sqoop/manager/oracle/OraOopJdbcUrl.java | 72 ++++++++++++-------- .../manager/oracle/OraOopManagerFactory.java | 30 +++++--- .../sqoop/manager/oracle/OraOopUtilities.java | 35 ++++++++-- 7 files changed, 111 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/docs/user/connectors.txt ---------------------------------------------------------------------- diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt index c04900f..062201d 100644 --- a/src/docs/user/connectors.txt +++ b/src/docs/user/connectors.txt @@ -692,13 +692,16 @@ The Sqoop +--connect+ parameter defines the Oracle instance or Oracle RAC to connect to. It is required with all Sqoop import and export commands. Data Connector for Oracle and Hadoop expects the associated connection string -to be of a specific format dependent on whether the Oracle SID or Service -is defined. +to be of a specific format dependent on whether the Oracle SID, Service +or TNS name is defined. The TNS name based URL scheme can be used to enable +authentication using Oracle wallets. +--connect jdbc:oracle:thin:@OracleServer:OraclePort:OracleSID+ +--connect jdbc:oracle:thin:@//OracleServer:OraclePort/OracleService+ ++--connect jdbc:oracle:thin:@TNSName+ + Connect to An Oracle Database Instance ++++++++++++++++++++++++++++++++++++++ @@ -720,6 +723,8 @@ with other drivers such as OCI. |The Oracle instance. |+OracleService+ |The Oracle Service. +|+TNSName+ +|The TNS name for the entry describing the connection to the Oracle server. |=============================================================================== [NOTE] http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/java/org/apache/sqoop/manager/OracleManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/OracleManager.java b/src/java/org/apache/sqoop/manager/OracleManager.java index 32cb303..69b613f 100644 --- a/src/java/org/apache/sqoop/manager/OracleManager.java +++ b/src/java/org/apache/sqoop/manager/OracleManager.java @@ -345,7 +345,7 @@ public class OracleManager return connection; } - public String getSessionUser(Connection conn) { + public static String getSessionUser(Connection conn) { Statement stmt = null; ResultSet rset = null; String user = null; @@ -380,6 +380,9 @@ public class OracleManager } } } + if (user == null) { + throw new RuntimeException("Unable to get current session user"); + } return user; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java b/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java index 302849c..3879453 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopConnManager.java @@ -31,6 +31,7 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.sqoop.manager.OracleManager; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.manager.ExportJobContext; @@ -96,7 +97,10 @@ public class OraOopConnManager extends GenericJdbcManager { Connection connection = OracleConnectionFactory.createOracleJdbcConnection(this .getDriverClass(), connectStr, username, password, additionalProps); - + if (username == null) { + username = OracleManager.getSessionUser(connection); + } + OraOopUtilities.setCurrentSessionUser(username); return connection; } @@ -627,4 +631,5 @@ public class OraOopConnManager extends GenericJdbcManager { .getJavaClassPath()); LOG.fatal(msg, ex); } + } http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/java/org/apache/sqoop/manager/oracle/OraOopConstants.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopConstants.java b/src/java/org/apache/sqoop/manager/oracle/OraOopConstants.java index 874ef02..26f1901 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopConstants.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopConstants.java @@ -399,6 +399,9 @@ public final class OraOopConstants { + " OR DATA_TYPE LIKE 'TIMESTAMP(%) WITH TIME ZONE'" + " OR DATA_TYPE LIKE 'TIMESTAMP(%) WITH LOCAL TIME ZONE'" + ")"; + // Query to get current logged on user + public static final String QUERY_GET_SESSION_USER = "SELECT USER FROM DUAL"; + // public static final int[] SUPPORTED_ORACLE_DATA_TYPES = { // oracle.jdbc.OracleTypes.BIT // -7; // ,oracle.jdbc.OracleTypes.TINYINT // -6; http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/java/org/apache/sqoop/manager/oracle/OraOopJdbcUrl.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopJdbcUrl.java b/src/java/org/apache/sqoop/manager/oracle/OraOopJdbcUrl.java index 8502b79..db7ea0c 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopJdbcUrl.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopJdbcUrl.java @@ -89,6 +89,7 @@ public class OraOopJdbcUrl { /* * The format of an Oracle jdbc URL is one of: + * jdbc:oracle:<driver-type>:@tnsname - for tnsname based login * jdbc:oracle:<driver-type>:@<host>:<port>:<sid> * jdbc:oracle:<driver-type>:@<host>:<port>/<service> * jdbc:oracle:<driver-type>:@<host>:<port>/<service>?<parameters> @@ -105,11 +106,12 @@ public class OraOopJdbcUrl { } // Check we can proceed... - if (jdbcFragments.length < 5 || jdbcFragments.length > 6) { + if (jdbcFragments.length < 4 || jdbcFragments.length > 6) { throw new JdbcOracleThinConnectionParsingError( String.format( - "There should be 5 or 6 colon-separated pieces of data in the JDBC " - + "URL, such as:\n\tjdbc:oracle:<driver-type>:@<host>:<port>:<sid>\n" + "There should be 4, 5 or 6 colon-separated pieces of data in the " + + "JDBC URL, such as:\n\tjdbc:oracle:<driver-type>:@tnsname\n" + + "\tjdbc:oracle:<driver-type>:@<host>:<port>:<sid>\n" + "\tjdbc:oracle:<driver-type>:@<host>:<port>/<service>\n" + "\tjdbc:oracle:<driver-type>:@<host>:<port>/<service>?<parameters>\n" + "The JDBC URL specified was:\n" @@ -156,6 +158,8 @@ public class OraOopJdbcUrl { } String portStr = ""; + String tnsName = ""; + switch (jdbcFragments.length) { case 6: // jdbc:oracle:<driver-type>:@<host>:<port>:<sid> @@ -176,41 +180,49 @@ public class OraOopJdbcUrl { service = portAndService[1].trim(); break; + case 4: + // jdbc:oracle:<driver-type>:@tnsname + tnsName = jdbcFragments[3].trim(); + break; + default: throw new JdbcOracleThinConnectionParsingError("Internal error parsing " + "JDBC connection string."); } - if (portStr.isEmpty()) { - throw new JdbcOracleThinConnectionParsingError( - "The fifth item in the colon-separated JDBC URL (the port) must not " - + "be empty."); - } - - try { - port = Integer.parseInt(portStr); - } catch (NumberFormatException ex) { - throw new JdbcOracleThinConnectionParsingError( - String - .format( - "The fifth item in the colon-separated JDBC URL (the port) " - + "must be a valid number.\n" - + "\"%s\" could not be parsed as an integer.", portStr)); - } - - if (port <= 0) { - throw new JdbcOracleThinConnectionParsingError( - String - .format( - "The fifth item in the colon-separated JDBC URL (the port) " - + "must be greater than zero.\n" - + "\"%s\" was specified.", portStr)); + if (jdbcFragments.length > 4) { + if (portStr.isEmpty()) { + throw new JdbcOracleThinConnectionParsingError( + "The fifth item in the colon-separated JDBC URL (the port) must not" + + " be empty."); + } + + try { + port = Integer.parseInt(portStr); + } catch (NumberFormatException ex) { + throw new JdbcOracleThinConnectionParsingError( + String + .format( + "The fifth item in the colon-separated JDBC URL (the port) " + + "must be a valid number.\n" + + "\"%s\" could not be parsed as an integer.", portStr)); + } + + if (port <= 0) { + throw new JdbcOracleThinConnectionParsingError( + String + .format( + "The fifth item in the colon-separated JDBC URL (the port) " + + "must be greater than zero.\n" + + "\"%s\" was specified.", portStr)); + } } - if (sid == null && service == null) { + if (sid == null && service == null && tnsName == null) { throw new JdbcOracleThinConnectionParsingError( "The JDBC URL does not contain a SID or SERVICE. The URL should look " - + "like one of these:\n\tjdbc:oracle:<driver-type>:@<host>:<port>:<sid>\n" + + "like one of these:\n\tjdbc:oracle:<driver-type>:@tnsname\n" + + "\tjdbc:oracle:<driver-type>:@<host>:<port>:<sid>\n" + "\tjdbc:oracle:<driver-type>:@<host>:<port>/<service>\n" + "\tjdbc:oracle:<driver-type>:@<host>:<port>/<service>?<parameters>\n" + "\tjdbc:oracle:<driver-type>:@//<host>:<port>/<service>\n" @@ -220,7 +232,7 @@ public class OraOopJdbcUrl { // Remove the "@" prefix of the hostname JdbcOracleThinConnection result = new JdbcOracleThinConnection(hostName.replaceFirst("^[@][/]{0,2}", "") - , port, sid, service); + , port, sid, service, tnsName.replaceFirst("^[@][/]{0,2}", "")); return result; } http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/java/org/apache/sqoop/manager/oracle/OraOopManagerFactory.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopManagerFactory.java b/src/java/org/apache/sqoop/manager/oracle/OraOopManagerFactory.java index fcf4a4d..9d75666 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopManagerFactory.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopManagerFactory.java @@ -34,6 +34,8 @@ import com.cloudera.sqoop.SqoopOptions.IncrementalMode; import com.cloudera.sqoop.manager.ConnManager; import com.cloudera.sqoop.manager.ManagerFactory; import com.cloudera.sqoop.metastore.JobData; + +import org.apache.sqoop.manager.OracleManager; import org.apache.sqoop.manager.oracle.OraOopOutputFormatUpdate.UpdateMode; import org.apache.sqoop.manager.oracle.OraOopUtilities. JdbcOracleThinConnectionParsingError; @@ -319,6 +321,7 @@ public class OraOopManagerFactory extends ManagerFactory { int jdbcPort = 0; String jdbcSid = ""; String jdbcService = ""; + String jdbcTnsName = ""; try { OraOopJdbcUrl oraOopJdbcUrl = new OraOopJdbcUrl(jdbcConnectStr); @@ -328,6 +331,7 @@ public class OraOopManagerFactory extends ManagerFactory { jdbcPort = jdbcConnection.getPort(); jdbcSid = jdbcConnection.getSid(); jdbcService = jdbcConnection.getService(); + jdbcTnsName = jdbcConnection.getTnsName(); } catch (JdbcOracleThinConnectionParsingError ex) { LOG.info(String.format( "Unable to parse the JDBC connection URL \"%s\" as a connection " @@ -362,18 +366,19 @@ public class OraOopManagerFactory extends ManagerFactory { jobData); } } else { - generateJdbcConnectionUrlsBySidOrService(jdbcHost, jdbcPort, jdbcSid, - jdbcService, jobData); + generateJdbcConnectionUrlsByTnsnameSidOrService(jdbcHost, jdbcPort, + jdbcSid, jdbcService, jdbcTnsName, jobData); } } - private void generateJdbcConnectionUrlsBySidOrService(String hostName, - int port, String sid, String serviceName, JobData jobData) { + private void generateJdbcConnectionUrlsByTnsnameSidOrService(String hostName, + int port, String sid, String serviceName, String tnsName, JobData jobData) { String jdbcUrl = null; - - if (sid != null && !sid.isEmpty()) { + if (tnsName != null && !tnsName.isEmpty()) { + jdbcUrl = OraOopUtilities.generateOracleTnsNameJdbcUrl(tnsName); + } else if (sid != null && !sid.isEmpty()) { jdbcUrl = OraOopUtilities.generateOracleSidJdbcUrl(hostName, port, sid); } else { jdbcUrl = @@ -437,7 +442,7 @@ public class OraOopManagerFactory extends ManagerFactory { jdbcActiveInstanceThinConnection = new OraOopUtilities.JdbcOracleThinConnection( activeInstance.getHostName(), - jdbcPort, activeInstance.getInstanceName(), ""); + jdbcPort, activeInstance.getInstanceName(), "", ""); if (testDynamicallyGeneratedOracleRacInstanceConnection( jdbcActiveInstanceThinConnection.toString(), jobData @@ -533,6 +538,8 @@ public class OraOopManagerFactory extends ManagerFactory { Configuration conf = jobData.getSqoopOptions().getConf(); String mapperJdbcUrlPropertyName = OraOopUtilities.getMapperJdbcUrlPropertyName(mapperIdx, conf); + LOG.debug("Setting mapper url " + mapperJdbcUrlPropertyName + " = " + + jdbcUrl); conf.set(mapperJdbcUrlPropertyName, jdbcUrl); } @@ -821,9 +828,14 @@ public class OraOopManagerFactory extends ManagerFactory { String exportTableTemplate = conf.get(OraOopConstants.ORAOOP_EXPORT_CREATE_TABLE_TEMPLATE, ""); + + String user = sqoopOptions.getUsername(); + if (user == null) { + user = OracleManager.getSessionUser(connection); + } + OracleTable templateTableContext = - OraOopUtilities.decodeOracleTableName(sqoopOptions.getUsername(), - exportTableTemplate); + OraOopUtilities.decodeOracleTableName(user, exportTableTemplate); boolean noLoggingOnNewTable = conf.getBoolean(OraOopConstants.ORAOOP_EXPORT_CREATE_TABLE_NO_LOGGING, http://git-wip-us.apache.org/repos/asf/sqoop/blob/1e2bc6e7/src/java/org/apache/sqoop/manager/oracle/OraOopUtilities.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/manager/oracle/OraOopUtilities.java b/src/java/org/apache/sqoop/manager/oracle/OraOopUtilities.java index 473a5ae..e81588c 100644 --- a/src/java/org/apache/sqoop/manager/oracle/OraOopUtilities.java +++ b/src/java/org/apache/sqoop/manager/oracle/OraOopUtilities.java @@ -38,14 +38,13 @@ import org.apache.hadoop.fs.Path; import org.apache.log4j.Category; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.apache.sqoop.manager.oracle.OraOopOutputFormatInsert.InsertMode; +import org.apache.sqoop.manager.oracle.OraOopOutputFormatUpdate.UpdateMode; import com.cloudera.sqoop.Sqoop; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.mapreduce.ExportJobBase; -import org.apache.sqoop.manager.oracle.OraOopOutputFormatInsert.InsertMode; -import org.apache.sqoop.manager.oracle.OraOopOutputFormatUpdate.UpdateMode; - /** * Utilities used by OraOop. */ @@ -54,6 +53,8 @@ public final class OraOopUtilities { private OraOopUtilities() { } + private static String currentSessionUser = null; + /** * Used for testing purposes - can get OraOop to call a class to run a report * on various performance metrics. @@ -104,6 +105,10 @@ public final class OraOopUtilities { // "schema"."table" String[] tableStrings = tableStr.split("\""); + if (oracleConnectionUserName == null) { + oracleConnectionUserName = currentSessionUser; + } + switch (tableStrings.length) { case 1: // <- table or schema.table @@ -748,17 +753,23 @@ public final class OraOopUtilities { private int port; private String sid; private String service; + private String tnsName; public JdbcOracleThinConnection(String host, int port, String sid, - String service) { + String service, String tnsName) { this.host = host; this.port = port; this.sid = sid; this.service = service; + this.tnsName = tnsName; } @Override public String toString() { + // Use tnsName if it is available + if (this.tnsName != null && !this.tnsName.isEmpty()) { + return String.format("jdbc:oracle:thin:@%s", tnsName); + } // Use the SID if it's available... if (this.sid != null && !this.sid.isEmpty()) { @@ -774,8 +785,8 @@ public final class OraOopUtilities { } throw new RuntimeException( - "Unable to generate a JDBC URL, as no SID or SERVICE has been " - + "provided."); + "Unable to generate a JDBC URL, as no TNS name, SID or SERVICE " + + "has been provided."); } @@ -794,6 +805,10 @@ public final class OraOopUtilities { public String getService() { return service; } + + public String getTnsName() { + return tnsName; + } } /** @@ -844,6 +859,10 @@ public final class OraOopUtilities { port, serviceName); } + public static String generateOracleTnsNameJdbcUrl(String tnsName) { + return String.format("jdbc:oracle:thin:@%s", tnsName); + } + public static String getMapperJdbcUrlPropertyName(int mapperId, org.apache.hadoop.conf.Configuration conf) { @@ -1457,5 +1476,7 @@ public final class OraOopUtilities { + "file:///dev/urandom - Oracle connections may time out."); } } - + public static void setCurrentSessionUser(String user) { + currentSessionUser = user; + } }
