Repository: sqoop Updated Branches: refs/heads/trunk d03faf354 -> 6bfaa9d65
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6bfaa9d6/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java new file mode 100644 index 0000000..871d317 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/util/OracleData.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle.util; + +import java.net.URL; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.List; + +import org.apache.commons.io.IOUtils; +import org.apache.sqoop.manager.oracle.OraOopTestCase; + +/** + * Class to load an Oracle table with data based on configuration file. + */ +public final class OracleData { + private OracleData() { + } + + enum KeyType { + PRIMARY, UNIQUE + } + + private static ClassLoader classLoader; + static { + classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = OraOopTestCase.class.getClassLoader(); + } + } + + private static String getColumnList(List<OracleDataDefinition> columnList) { + StringBuilder result = new StringBuilder(); + String delim = ""; + for (OracleDataDefinition column : columnList) { + result.append(delim).append(column.getColumnName()).append(" ").append( + column.getDataType()); + delim = ",\n"; + } + return result.toString(); + } + + private static String + getDataExpression(List<OracleDataDefinition> columnList) { + StringBuilder result = new StringBuilder(); + for (OracleDataDefinition column : columnList) { + result.append("l_ret_rec.").append(column.getColumnName()).append(" := ") + .append(column.getDataExpression()).append(";\n"); + } + return result.toString(); + } + + private static void createPackageSpec(Connection conn, + OracleTableDefinition tableDefinition) throws Exception { + String pkgSql = + IOUtils.toString(classLoader.getResource( + "oraoop/pkg_tst_product_gen.psk").openStream()); + pkgSql = + pkgSql.replaceAll("\\$COLUMN_LIST", getColumnList(tableDefinition + .getColumnList())); + pkgSql = pkgSql.replaceAll("\\$TABLE_NAME", tableDefinition.getTableName()); + PreparedStatement stmt = conn.prepareStatement(pkgSql); + stmt.execute(); + } + + private static void createPackageBody(Connection conn, + OracleTableDefinition tableDefinition) throws Exception { + String pkgSql = + IOUtils.toString(classLoader.getResource( + "oraoop/pkg_tst_product_gen.pbk").openStream()); + pkgSql = + pkgSql.replaceAll("\\$COLUMN_LIST", getColumnList(tableDefinition + .getColumnList())); + pkgSql = pkgSql.replaceAll("\\$TABLE_NAME", tableDefinition.getTableName()); + pkgSql = + pkgSql.replaceAll("\\$DATA_EXPRESSION_LIST", + getDataExpression(tableDefinition.getColumnList())); + pkgSql = + pkgSql.replaceAll("\\$PARTITION_CLAUSE", tableDefinition + .getPartitionClause()); + PreparedStatement stmt = conn.prepareStatement(pkgSql); + stmt.execute(); + } + + private static void createKey(Connection conn, KeyType keyType, + OracleTableDefinition tableDefinition) throws Exception { + List<String> columns = null; + switch (keyType) { + case PRIMARY: + columns = tableDefinition.getPrimaryKeyColumns(); + break; + case UNIQUE: + columns = tableDefinition.getUniqueKeyColumns(); + break; + default: + throw new RuntimeException("Missing key type."); + } + if (columns != null && columns.size() > 0) { + StringBuilder keyColumnList = new StringBuilder(); + String delim = ""; + for (String column : columns) { + keyColumnList.append(delim).append(column); + delim = ","; + } + String keySql = + "alter table \"$TABLE_NAME\" add constraint \"$TABLE_NAME_" + + ((keyType == KeyType.PRIMARY) ? "PK\" primary key" + : "UK\" unique") + "($PK_COLUMN_LIST) " + + "using index (create unique index \"$TABLE_NAME_" + + ((keyType == KeyType.PRIMARY) ? "PK\"" : "UK\"") + + " on \"$TABLE_NAME\"($PK_COLUMN_LIST) " + "parallel nologging)"; + keySql = + keySql.replaceAll("\\$TABLE_NAME", tableDefinition.getTableName()); + keySql = keySql.replaceAll("\\$PK_COLUMN_LIST", keyColumnList.toString()); + PreparedStatement stmt = conn.prepareStatement(keySql); + stmt.execute(); + } + } + + public static int getParallelProcesses(Connection conn) throws Exception { + PreparedStatement stmt = + conn.prepareStatement("SELECT cc.value value" + + "\n" + + "FROM" + + "\n" + + " (SELECT to_number(value) value" + + "\n" + + " FROM v$parameter" + + "\n" + + " WHERE name='parallel_max_servers'" + + "\n" + + " ) pms," + + "\n" + + " (SELECT to_number(value) value" + + "\n" + + " FROM v$parameter" + + "\n" + + " WHERE name='parallel_threads_per_cpu'" + + "\n" + + " ) ptpc," + + "\n" + + " (SELECT to_number(value) value FROM v$parameter " + + " WHERE name='cpu_count'" + + "\n" + " ) cc"); + ResultSet res = stmt.executeQuery(); + res.next(); + return res.getInt(1); + } + + public static void createTable(Connection conn, + OracleTableDefinition tableDefinition, int parallelDegree, + int rowsPerSlave) throws Exception { + createPackageSpec(conn, tableDefinition); + createPackageBody(conn, tableDefinition); + + CallableStatement procStmt = + conn.prepareCall("begin \"PKG_ODG_" + tableDefinition.getTableName() + + "\".prc_load_table(?,?); end;"); + procStmt.setInt(1, parallelDegree); + procStmt.setInt(2, rowsPerSlave); + procStmt.execute(); + + createKey(conn, KeyType.PRIMARY, tableDefinition); + createKey(conn, KeyType.UNIQUE, tableDefinition); + } + + public static void createTable(Connection conn, String fileName, + int parallelDegree, int rowsPerSlave) throws Exception { + URL file = classLoader.getResource("oraoop/" + fileName); + OracleTableDefinition tableDefinition = new OracleTableDefinition(file); + createTable(conn, tableDefinition, parallelDegree, rowsPerSlave); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6bfaa9d6/src/test/org/apache/sqoop/manager/oracle/util/OracleDataDefinition.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/OracleDataDefinition.java b/src/test/org/apache/sqoop/manager/oracle/util/OracleDataDefinition.java new file mode 100644 index 0000000..1256838 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/util/OracleDataDefinition.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle.util; + +/** + * Holds column definition for generated Oracle table. + */ +public class OracleDataDefinition { + + private String columnName; + private String dataType; + private String dataExpression; + + public OracleDataDefinition(String columnName, String dataType, + String dataExpression) { + this.columnName = columnName; + this.dataType = dataType; + this.dataExpression = dataExpression; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String newColumnName) { + this.columnName = newColumnName; + } + + public String getDataExpression() { + return dataExpression; + } + + public void setDataExpression(String newDataExpression) { + this.dataExpression = newDataExpression; + } + + public String getDataType() { + return dataType; + } + + public void setDataType(String newDataType) { + this.dataType = newDataType; + } + + @Override + public String toString() { + return this.getColumnName(); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6bfaa9d6/src/test/org/apache/sqoop/manager/oracle/util/OracleTableDefinition.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/OracleTableDefinition.java b/src/test/org/apache/sqoop/manager/oracle/util/OracleTableDefinition.java new file mode 100644 index 0000000..5a8c42c --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/util/OracleTableDefinition.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle.util; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Holds table definition for generated Oracle table. + */ +public class OracleTableDefinition { + + private String tableName; + private List<OracleDataDefinition> columnList = + new ArrayList<OracleDataDefinition>(); + private List<String> primaryKeyColumns = new ArrayList<String>(); + private List<String> uniqueKeyColumns = new ArrayList<String>(); + private String partitionClause; + + public List<String> getUniqueKeyColumns() { + return uniqueKeyColumns; + } + + public void setUniqueKeyColumns(List<String> newUniqueKeyColumns) { + this.uniqueKeyColumns = newUniqueKeyColumns; + } + + public List<String> getPrimaryKeyColumns() { + return primaryKeyColumns; + } + + public void setPrimaryKeyColumns(List<String> newPrimaryKeyColumns) { + this.primaryKeyColumns = newPrimaryKeyColumns; + } + + public List<OracleDataDefinition> getColumnList() { + return columnList; + } + + public void setColumnList(List<OracleDataDefinition> newColumnList) { + this.columnList = newColumnList; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String newTableName) { + this.tableName = newTableName; + } + + public String getPartitionClause() { + return partitionClause == null ? "" : partitionClause; + } + + public void setPartitionClause(String newPartitionClause) { + this.partitionClause = newPartitionClause; + } + + public OracleTableDefinition() { + + } + + public OracleTableDefinition(URL url) { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(new File(url.toURI())); + + Element table = doc.getDocumentElement(); + this.tableName = + table.getElementsByTagName("name").item(0).getChildNodes().item(0) + .getNodeValue(); + NodeList columns = table.getElementsByTagName("column"); + for (int i = 0; i < columns.getLength(); i++) { + Node columnNode = columns.item(i); + if (columnNode.getNodeType() == Node.ELEMENT_NODE) { + Element columnElement = (Element) columnNode; + String name = + columnElement.getElementsByTagName("name").item(0) + .getChildNodes().item(0).getNodeValue(); + String dataType = + columnElement.getElementsByTagName("dataType").item(0) + .getChildNodes().item(0).getNodeValue(); + String dataExpression = + columnElement.getElementsByTagName("dataExpression").item(0) + .getChildNodes().item(0).getNodeValue(); + this.columnList.add(new OracleDataDefinition(name, dataType, + dataExpression)); + } + } + + NodeList primaryKeyColumnsNodeList = + table.getElementsByTagName("primaryKeyColumn"); + for (int i = 0; i < primaryKeyColumnsNodeList.getLength(); i++) { + Node primaryKeyColumnNode = primaryKeyColumnsNodeList.item(i); + if (primaryKeyColumnNode.getNodeType() == Node.ELEMENT_NODE) { + Element primaryKeyColumnElement = (Element) primaryKeyColumnNode; + this.primaryKeyColumns.add(primaryKeyColumnElement.getChildNodes() + .item(0).getNodeValue()); + } + } + + NodeList uniqueKeyColumnsNodeList = + table.getElementsByTagName("uniqueKeyColumn"); + for (int i = 0; i < uniqueKeyColumnsNodeList.getLength(); i++) { + Node uniqueKeyColumnNode = uniqueKeyColumnsNodeList.item(i); + if (uniqueKeyColumnNode.getNodeType() == Node.ELEMENT_NODE) { + Element uniqueKeyColumnElement = (Element) uniqueKeyColumnNode; + this.uniqueKeyColumns.add(uniqueKeyColumnElement.getChildNodes() + .item(0).getNodeValue()); + } + } + + Node partitionClauseNode = + table.getElementsByTagName("partitionClause").item(0); + if (partitionClauseNode != null) { + this.partitionClause = + partitionClauseNode.getChildNodes().item(0).getNodeValue(); + } + } catch (Exception e) { + throw new RuntimeException("Could not load table configuration", e); + } + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6bfaa9d6/src/test/org/apache/sqoop/manager/oracle/util/RowIdGenerator.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/RowIdGenerator.java b/src/test/org/apache/sqoop/manager/oracle/util/RowIdGenerator.java new file mode 100644 index 0000000..bb3a71b --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/util/RowIdGenerator.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle.util; + +import java.lang.reflect.Constructor; +import java.nio.charset.Charset; +import java.sql.RowId; + +/** + * Generates ROWID test data. ROWIDs are represented by 18 ASCII encoded + * characters from the set A-Za-z0-9/+ + * + * Generated ROWIDs are unlikely to represent actual rows in any Oracle + * database, so should be used for import/export tests only, and not used to + * reference data. + */ +public class RowIdGenerator extends OraOopTestDataGenerator<RowId> { + private static final String VALID_CHARS = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789/+"; + private static final int LENGTH = 18; + private static Class<?> rowIdClass; + private static Constructor<?> rowIdConstructor; + + static { + try { + rowIdClass = Class.forName("oracle.sql.ROWID"); + rowIdConstructor = rowIdClass.getConstructor(byte[].class); + } catch (Exception e) { + throw new RuntimeException( + "Problem getting Oracle JDBC methods via reflection.", e); + } + } + + @Override + public RowId next() { + try { + StringBuffer sb = new StringBuffer(); + while (sb.length() < LENGTH) { + sb.append(VALID_CHARS.charAt(rng.nextInt(VALID_CHARS.length()))); + } + return (RowId) rowIdConstructor.newInstance(sb.toString().getBytes( + Charset.forName("US-ASCII"))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6bfaa9d6/src/test/org/apache/sqoop/manager/oracle/util/TimestampGenerator.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/TimestampGenerator.java b/src/test/org/apache/sqoop/manager/oracle/util/TimestampGenerator.java new file mode 100644 index 0000000..b2ce366 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/util/TimestampGenerator.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle.util; + +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.GregorianCalendar; + +/** + * Generates test data for Oracle DATE columns. Generated Timestamps are between + * 4711BC and 9999AD. + */ +public class TimestampGenerator extends OraOopTestDataGenerator<Timestamp> { + private static final int NANOS_DIGITS = 9; + private static final int MIN_YEAR = -4711; + private static final int MAX_YEAR = 9999; + private final int precision; + private final Calendar cal = Calendar.getInstance(); + + /** + * Create a TimestampGenerator that will generate Timestamps with a given + * precision. + * + * @param precision + * Number of decimal digits after the decimal point in the seconds of + * generated Timestamps. + */ + public TimestampGenerator(int precision) { + this.precision = precision; + } + + @Override + public Timestamp next() { + cal.clear(); + cal.set(Calendar.YEAR, MIN_YEAR + rng.nextInt(MAX_YEAR - MIN_YEAR + 1)); + cal.set(Calendar.DAY_OF_YEAR, 1 + rng.nextInt(cal + .getActualMaximum(Calendar.DAY_OF_YEAR))); + cal.set(Calendar.HOUR_OF_DAY, rng.nextInt(24)); + cal.set(Calendar.MINUTE, rng.nextInt(60)); + cal.set(Calendar.SECOND, rng.nextInt( + cal.getActualMaximum(Calendar.SECOND))); + // Workaround for oracle jdbc bugs related to BC leap years + if (cal.get(Calendar.ERA) == GregorianCalendar.BC + && cal.get(Calendar.MONTH) == 1 && cal.get(Calendar.DAY_OF_MONTH) >= 28) { + return next(); + } + Timestamp timestamp = new Timestamp(cal.getTimeInMillis()); + if (precision > 0) { + int nanos = rng.nextInt((int) Math.pow(10, precision)); + timestamp.setNanos(nanos * (int) Math.pow(10, NANOS_DIGITS - precision)); + } + return timestamp; + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6bfaa9d6/src/test/org/apache/sqoop/manager/oracle/util/URIGenerator.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/oracle/util/URIGenerator.java b/src/test/org/apache/sqoop/manager/oracle/util/URIGenerator.java new file mode 100644 index 0000000..63fec01 --- /dev/null +++ b/src/test/org/apache/sqoop/manager/oracle/util/URIGenerator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.manager.oracle.util; + +/** + * Generates test data for Oracle UriType columns. Generated Strings can be cast + * to URITypes with sys.UriFactory.getUri(value). Generated strings may be + * detect as any of HTTPURIType, DBURIType or XDBURIType. + * + * Generated URIs are unlikely to resolve successfully, so should be used for + * import/export tests only, and not used to reference data. + */ +public class URIGenerator extends OraOopTestDataGenerator<String> { + private static final int MIN_LENGTH = 15; + private static final int MAX_LENGTH = 30; + + @Override + public String next() { + StringBuilder sb = new StringBuilder(); + switch (rng.nextInt(3)) { + case 0: // Generate a String that will detect as an HTTPURIType + sb.append("http://";); + break; + case 1: // Generate a String that will detect as an DBURIType + sb.append("/oradb/"); + break; + case 2: // Generate a String that will detect as an XDBURIType + break; + default: + throw new RuntimeException("Invalid number generated."); + } + + int length = + sb.length() + MIN_LENGTH + rng.nextInt(MAX_LENGTH - MIN_LENGTH + 1); + while (sb.length() < length) { + sb.append(Character.toChars(rng.nextInt(128))); + } + return sb.toString(); + } + +}
