Updated Branches: refs/heads/sqoop2 7ca9b4468 -> d9465bba2
SQOOP-788 Import sometimes duplicate some data (Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d9465bba Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d9465bba Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d9465bba Branch: refs/heads/sqoop2 Commit: d9465bba216372f053ba9c652b8758f5941b3ead Parents: 7ca9b44 Author: Bilung Lee <[email protected]> Authored: Tue Dec 18 15:52:46 2012 -0800 Committer: Bilung Lee <[email protected]> Committed: Tue Dec 18 15:52:46 2012 -0800 ---------------------------------------------------------------------- .../connector/jdbc/GenericJdbcImportExtractor.java | 2 +- .../sqoop/job/etl/HdfsSequenceExportExtractor.java | 1 + .../sqoop/job/etl/HdfsTextExportExtractor.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/d9465bba/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java ---------------------------------------------------------------------- diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java index b856ce6..df78755 100644 --- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java +++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java @@ -59,8 +59,8 @@ public class GenericJdbcImportExtractor extends Extractor { try { ResultSetMetaData metaData = resultSet.getMetaData(); int column = metaData.getColumnCount(); - Object[] array = new Object[column]; while (resultSet.next()) { + Object[] array = new Object[column]; for (int i = 0; i< column; i++) { array[i] = resultSet.getObject(i+1); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/d9465bba/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java index 16afcdb..3a04e59 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsSequenceExportExtractor.java @@ -84,6 +84,7 @@ public class HdfsSequenceExportExtractor extends Extractor { boolean hasNext = filereader.next(line); while (hasNext) { datawriter.writeCsvRecord(line.toString()); + line = new Text(); hasNext = filereader.next(line); if(filereader.getPosition() >= end && filereader.syncSeen()) { break; http://git-wip-us.apache.org/repos/asf/sqoop/blob/d9465bba/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java index 8055140..e00d428 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/etl/HdfsTextExportExtractor.java @@ -104,11 +104,11 @@ public class HdfsTextExportExtractor extends Extractor { // one extra line is read in previous split start += filereader.readLine(new Text(), 0); } - Text line = new Text(); int size; LOG.info("Start position: " + String.valueOf(start)); long next = start; while (next <= end) { + Text line = new Text(); size = filereader.readLine(line, Integer.MAX_VALUE); if (size == 0) { break;
