http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestCodecMap.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/io/TestCodecMap.java b/src/test/org/apache/sqoop/io/TestCodecMap.java new file mode 100644 index 0000000..e719218 --- /dev/null +++ b/src/test/org/apache/sqoop/io/TestCodecMap.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.io; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.GzipCodec; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Rule; + +import org.junit.rules.ExpectedException; + +/** + * Test looking up codecs by name. + */ +public class TestCodecMap { + + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private void verifyCodec(Class<?> c, String codecName) + throws UnsupportedCodecException { + CompressionCodec codec = CodecMap.getCodec(codecName, new Configuration()); + assertEquals(codec.getClass(), c); + } + + @Test + public void testGetCodecNames() { + // gzip is picked up from Hadoop defaults + assertTrue(CodecMap.getCodecNames().contains("gzip")); + } + + @Test + public void testGetCodec() throws IOException { + verifyCodec(GzipCodec.class, "gzip"); + verifyCodec(GzipCodec.class, "Gzip"); + verifyCodec(GzipCodec.class, "GZIP"); + verifyCodec(GzipCodec.class, "gzipcodec"); + verifyCodec(GzipCodec.class, "GzipCodec"); + verifyCodec(GzipCodec.class, "GZIPCODEC"); + verifyCodec(GzipCodec.class, "org.apache.hadoop.io.compress.GzipCodec"); + } + + @Test + public void testGetShortName() throws UnsupportedCodecException { + verifyShortName("gzip", "org.apache.hadoop.io.compress.GzipCodec"); + verifyShortName("default", "org.apache.hadoop.io.compress.DefaultCodec"); + + thrown.expect(UnsupportedCodecException.class); + thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException with invalid codec name during getting " + + "short codec name"); + verifyShortName("NONE", "bogus"); + } + + private void verifyShortName(String expected, String codecName) + throws UnsupportedCodecException { + assertEquals(expected, + CodecMap.getCodecShortNameByName(codecName, new Configuration())); + } + + @Test + public void testUnrecognizedCodec() throws UnsupportedCodecException { + thrown.expect(UnsupportedCodecException.class); + thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException with invalid codec name"); + CodecMap.getCodec("bogus", new Configuration()); + } + +}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestLobFile.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/io/TestLobFile.java b/src/test/org/apache/sqoop/io/TestLobFile.java new file mode 100644 index 0000000..2bc95f2 --- /dev/null +++ b/src/test/org/apache/sqoop/io/TestLobFile.java @@ -0,0 +1,642 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.io; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.net.URI; +import java.nio.CharBuffer; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CryptoOutputStream; +import org.apache.hadoop.crypto.JceAesCtrCryptoCodec; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Before; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Answers.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.withSettings; + +import org.junit.Rule; +import org.junit.rules.ExpectedException; + +/** + * Test the LobFile reader/writer implementation. + */ +public class TestLobFile { + + public static final Log LOG = LogFactory.getLog( + TestLobFile.class.getName()); + + public static final Path TEMP_BASE_DIR; + + static { + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + if (!tmpDir.endsWith(File.separator)) { + tmpDir = tmpDir + File.separator; + } + + TEMP_BASE_DIR = new Path(new Path(tmpDir), "lobtest"); + } + + private Configuration conf; + private FileSystem fs; + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + conf.set("fs.default.name", "file:///"); + + fs = FileSystem.getLocal(conf); + fs.mkdirs(TEMP_BASE_DIR); + } + + private long[] writeClobFile(Path p, String codec, + String... records) throws Exception { + if (fs.exists(p)) { + fs.delete(p, false); + } + + // memorize the offsets of each record we write. + long[] offsets = new long[records.length]; + + // Create files with four entries per index segment. + LobFile.Writer writer = LobFile.create(p, conf, true, codec, 4); + + int i = 0; + for (String r : records) { + offsets[i++] = writer.tell(); + Writer w = writer.writeClobRecord(r.length()); + w.write(r); + w.close(); + } + + writer.close(); + return offsets; + } + + private void verifyClobFile(Path p, String... expectedRecords) + throws Exception { + + LobFile.Reader reader = LobFile.open(p, conf); + + int recNum = 0; + + while (reader.next()) { + // We should have a record of the same length as the expected one. + String expected = expectedRecords[recNum]; + assertTrue(reader.isRecordAvailable()); + assertEquals(expected.length(), reader.getRecordLen()); + Reader r = reader.readClobRecord(); + + // Read in the record and assert that we got enough characters out. + CharBuffer buf = CharBuffer.allocate(expected.length()); + int bytesRead = 0; + while (bytesRead < expected.length()) { + int thisRead = r.read(buf); + LOG.info("performed read of " + thisRead + " chars"); + if (-1 == thisRead) { + break; + } + + bytesRead += thisRead; + } + + LOG.info("Got record of " + bytesRead + " chars"); + assertEquals(expected.length(), bytesRead); + char[] charData = buf.array(); + String finalRecord = new String(charData); + assertEquals(expected, finalRecord); + + recNum++; + } + + // Check that we got everything. + assertEquals(expectedRecords.length, recNum); + + reader.close(); + + thrown.expect(IOException.class); + thrown.reportMissingExceptionWithMessage("Expected IOException calling next after close"); + reader.next(); + + // A second close shouldn't hurt anything. This should be a no-op. + reader.close(); + } + + private void runClobFileTest(Path p, String codec, + String... records) throws Exception { + writeClobFile(p, codec, records); + verifyClobFile(p, records); + fs.delete(p, false); + } + + @Test + public void testEmptyRecord() throws Exception { + runClobFileTest(new Path(TEMP_BASE_DIR, "empty.lob"), null); + } + + @Test + public void testSingleRecord() throws Exception { + runClobFileTest(new Path(TEMP_BASE_DIR, "single.lob"), + null, "this is a single record!"); + } + + @Test + public void testMultiRecords() throws Exception { + runClobFileTest(new Path(TEMP_BASE_DIR, "multi.lob"), + CodecMap.NONE, + "this is the first record", + "this is the second record. I assure you that this record is long.", + "yet one more record graces this file."); + } + + @Test + public void testMultiIndexSegments() throws Exception { + // Test that we can use multiple IndexSegments. + runClobFileTest(new Path(TEMP_BASE_DIR, "multi-index.lob"), + null, + "this is the first record", + "this is the second record. I assure you that this record is long.", + "record number three", + "last one in first index segment", + "first in the second index segment", + "yet one more record graces this file."); + } + + /** + * Run a test where we read only a fraction of the first record, + * but then read the second record completely. Verify that we + * can re-align on a record boundary correctly. This test requires + * at least 3 records. + * + * @param p the path to the file to create. + * @param firstLine the first line of the first reord + * @param records All of the records to write to the file. + */ + private void runLineAndRecordTest(Path p, String firstLine, + String... records) throws Exception { + + assertTrue("This test requires 3+ records", records.length > 2); + + writeClobFile(p, null, records); + + LobFile.Reader reader = LobFile.open(p, conf); + + // We should not yet be aligned. + assertFalse(reader.isRecordAvailable()); + assertTrue(reader.next()); + // Now we should be. + assertTrue(reader.isRecordAvailable()); + + // Read just one line from the record. + Reader r = reader.readClobRecord(); + BufferedReader br = new BufferedReader(r); + String line = br.readLine(); + assertEquals(firstLine, line); + + br.close(); + r.close(); + + // We should no longer be aligned on a record start. + assertFalse(reader.isRecordAvailable()); + + // We should now be able to get to record two. + assertTrue(reader.next()); + + // This should be nicely aligned even if the first record was not + // completely consumed by a client. + r = reader.readClobRecord(); + CharBuffer buf = CharBuffer.allocate(records[1].length()); + r.read(buf); + r.close(); + char[] chars = buf.array(); + String s = new String(chars); + assertEquals(records[1], s); + + // Close the reader before we consume the entire file. + reader.close(); + assertFalse(reader.isRecordAvailable()); + } + + @Test + public void testVeryShortRead() throws Exception { + // Read only a small fraction of a record, ensure that we can + // read the next record, even when we've left more than a 16-byte + // quantity in the readahead buffer. + + Path p = new Path(TEMP_BASE_DIR, "shortread.lob"); + final String FIRST_LINE = "line1"; + final String SECOND_LINE = + "This contains much more in the record than just one line."; + final String RECORD2 = "here is the second record."; + final String RECORD3 = "The 3rd record, which we won't actually read."; + + runLineAndRecordTest(p, FIRST_LINE, + FIRST_LINE + "\n" + SECOND_LINE, + RECORD2, + RECORD3); + + } + + @Test + public void testIncompleteOverread() throws Exception { + // Read most of the first record so that we partially consume the + // next record start mark; make sure we realign properly. + + Path p = new Path(TEMP_BASE_DIR, "longread.lob"); + final String FIRST_LINE = "this is a really long line of text to read!"; + final String SECOND_LINE = "not this."; + final String RECORD2 = "Here is yet another record to verify."; + final String RECORD3 = "Nobody cares about record 3."; + + runLineAndRecordTest(p, FIRST_LINE, + FIRST_LINE + "\n" + SECOND_LINE, + RECORD2, + RECORD3); + } + + @Test + public void testSeekToRecord() throws Exception { + // Seek past the first two records and read the third. + + Path p = new Path(TEMP_BASE_DIR, "seek.lob"); + String[] records = { + "this is the first record!", + "here comes record number two. It is a bit longer.", + "this is the third record. we can read it.", + }; + + // Write the file and memorize when the third record starts. + LobFile.Writer writer = LobFile.create(p, conf, true); + + int recNum = 0; + long rec3Start = 0; + for (String r : records) { + Writer w = writer.writeClobRecord(r.length()); + w.write(r); + w.close(); + writer.finishRecord(); + if (recNum == 1) { + rec3Start = writer.tell(); + LOG.info("Record three start: " + rec3Start); + } + recNum++; + } + + writer.close(); + + // Now reopen the file for read, seek to the third record, and get it. + LobFile.Reader reader = LobFile.open(p, conf); + reader.seek(rec3Start); + assertTrue(reader.next()); + assertTrue(reader.isRecordAvailable()); + assertEquals(rec3Start, reader.getRecordOffset()); + + Reader r = reader.readClobRecord(); + CharBuffer buf = CharBuffer.allocate(records[2].length()); + r.read(buf); + r.close(); + char[] chars = buf.array(); + String s = new String(chars); + assertEquals(records[2], s); + + r.close(); + reader.close(); + } + + + /** + * Verifies that the next record in the LobFile is the expected one. + */ + private void verifyNextRecord(LobFile.Reader reader, long expectedId, + String expectedRecord) throws Exception { + assertTrue(reader.next()); + assertTrue(reader.isRecordAvailable()); + assertEquals(expectedId, reader.getRecordId()); + + Reader r = reader.readClobRecord(); + CharBuffer buf = CharBuffer.allocate(expectedRecord.length()); + int bytesRead = 0; + while (bytesRead < expectedRecord.length()) { + int thisRead = r.read(buf); + if (-1 == thisRead) { + break; + } + + bytesRead += thisRead; + } + + LOG.info("Got record of " + bytesRead + " chars"); + assertEquals(expectedRecord.length(), bytesRead); + + char[] charData = buf.array(); + String finalRecord = new String(charData); + assertEquals(expectedRecord, finalRecord); + } + + @Test + public void testManySeeks() throws Exception { + // Test that we can do gymnastics with seeking between records. + + Path p = new Path(TEMP_BASE_DIR, "manyseeks.lob"); + + String[] records = { + "first record", + "second record", + "the third record", + "rec4 is the last in IndexSeg 0", + "rec5 is first in IndexSeg 1", + "rec6 is yet another record", + "rec7 is starting to feel boring", + "rec8 is at the end of seg 1", + "rec9 is all by itself in seg 2", + }; + + // Write the records to a file, save their offsets. + long[] offsets = writeClobFile(p, null, records); + + // Sanity check that we can stream the file. + verifyClobFile(p, records); + + // Open a handle to the file. + LobFile.Reader reader = LobFile.open(p, conf); + + // Seeking to offset 0 should return the first record. + reader.seek(0); + verifyNextRecord(reader, 0, records[0]); + + // Seek to the last item in the first IndexSegment. + reader.seek(offsets[3]); + verifyNextRecord(reader, 3, records[3]); + + // Seek to just ahead of that same record. + reader.seek(offsets[3] - 10); + verifyNextRecord(reader, 3, records[3]); + + // Seek (backwards) to the first record. + reader.seek(offsets[0]); + verifyNextRecord(reader, 0, records[0]); + + // Seek to first record in second IndexSegment. + reader.seek(offsets[4]); + verifyNextRecord(reader, 4, records[4]); + + // Move backwards. + reader.seek(0); + + // Seek to "no man's land" between last offset in first IndexSeg + // and the first offset in second IndexSegment. Result should be + // the first record in second InexSegment. + reader.seek(offsets[4] - 10); + verifyNextRecord(reader, 4, records[4]); + + // Seek to past the last record. No record should be returned. + reader.seek(offsets[8] + 4); + assertFalse("Found a record past last record start.", reader.next()); + + // Seek to somewhere in the middle of IndexSegment 0. + // This should recover just fine. + reader.seek(offsets[2]); + verifyNextRecord(reader, 2, records[2]); + + // Seek to last record in IndexSegment 1. + reader.seek(offsets[3] - 1); + verifyNextRecord(reader, 3, records[3]); + + // And make sure that iteration picks up naturally from there. + verifyNextRecord(reader, 4, records[4]); + + // Seek well past the end of the file. No record should be returned. + reader.seek(50000000); + assertFalse("Found a record past expected end-of-file", reader.next()); + + // Seek to somewhere in the index. + reader.seek(offsets[8] + 32); + assertFalse("Found a record past beginning of index", reader.next()); + + // Seek to the last record (exact hit). This is a singleton IndexSegment. + reader.seek(offsets[8]); + verifyNextRecord(reader, 8, records[8]); + + // Seek to no-man's-land ahead of last record. + reader.seek(offsets[8] - 3); + verifyNextRecord(reader, 8, records[8]); + + reader.close(); + } + + /** + * Verifies that a record to be read from a lob file has + * as many bytes as we expect, and that the bytes are what we + * expect them to be. Assumes that the bytes are such that + * input[i] == i + offset. + * + * @param reader the LobFile reader to consume data from + * @param expectedDeclaredLen the size we expect the LobFile to declare + * its record length as. + * @param expectedActualLen the true number of bytes we expect to read in + * the record. + * @param offset the offset amount for each of the elements of the array. + */ + private void verifyBlobRecord(LobFile.Reader reader, + long expectedDeclaredLen, long expectedActualLen, + int offset) throws Exception { + + assertTrue(reader.next()); + assertTrue(reader.isRecordAvailable()); + assertEquals(expectedDeclaredLen, reader.getRecordLen()); + + InputStream is = reader.readBlobRecord(); + + byte[] bytes = new byte[(int) expectedActualLen]; + int numRead = is.read(bytes); + assertEquals(expectedActualLen, numRead); + + for (int i = 0; i < numRead; i++) { + assertEquals(i + offset, (int) bytes[i]); + } + + is.close(); + } + + /** + * Write a binary record to a LobFile. This allows the declared length + * of the record to disagree with the actual length (the actual length + * should be >= the declared length). + * The record written will have values v[i] = i + offset. + * + * @param writer the LobFile writer to put the record into + * @param declaredLen the length value written into the file itself + * @param actualLen the true number of bytes to write + * @param offset an amount to adjust each record's byte values by. + */ + private void writeBlobRecord(LobFile.Writer writer, long declaredLen, + long actualLen, int offset) throws Exception { + OutputStream os = writer.writeBlobRecord(declaredLen); + for (int i = 0; i < actualLen; i++) { + os.write(i + offset); + } + + os.close(); + writer.finishRecord(); + } + + /** + * Verifies a number of records that all have the same declared + * and actual record lengths. + * + * @param p the path to the LobFile to open + * @param numRecords the number of records to expect + * @param declaredLen the declared length of each record in the file + * @param actualLen the true number of bytes we expect to read per record. + */ + private void verifyBlobRecords(Path p, int numRecords, + long declaredLen, long actualLen) throws Exception { + + LobFile.Reader reader = LobFile.open(p, conf); + for (int i = 0; i < numRecords; i++) { + verifyBlobRecord(reader, declaredLen, actualLen, i); + } + assertFalse(reader.next()); + reader.close(); + } + + @Test + public void testBinaryRecords() throws Exception { + // Write a BLOB file and read it all back. + + final long RECORD_LEN = 32; + final int NUM_RECORDS = 2; + Path p = new Path(TEMP_BASE_DIR, "binary.lob"); + LobFile.Writer writer = LobFile.create(p, conf); + + for (int i = 0; i < NUM_RECORDS; i++) { + writeBlobRecord(writer, RECORD_LEN, RECORD_LEN, i); + } + + writer.close(); + + // Now check the read-back on those records. + verifyBlobRecords(p, NUM_RECORDS, RECORD_LEN, RECORD_LEN); + } + + @Test + public void testOverLengthBinaryRecord() throws Exception { + // Write a record with a declared length shorter than the + // actual length, and read it back. + + final long ACTUAL_RECORD_LEN = 48; + final long DECLARED_RECORD_LEN = 32; + final int NUM_RECORDS = 2; + + Path p = new Path(TEMP_BASE_DIR, "overlength.lob"); + LobFile.Writer writer = LobFile.create(p, conf); + + for (int i = 0; i < NUM_RECORDS; i++) { + writeBlobRecord(writer, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN, i); + } + + writer.close(); + + // Now read them back. + verifyBlobRecords(p, NUM_RECORDS, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN); + } + + private void runCompressedTest(String codec) throws Exception { + LOG.info("Testing with codec: " + codec); + Path p = new Path(TEMP_BASE_DIR, "compressed-" + codec + ".lob"); + String[] records = { + "this is the first record, It should be compressed a lot!", + "record 2 record 2 record 2 record 2 2 2 2 2 2 2 2 2 2 2 2", + "and a third and a third yes this is the third", + }; + + runClobFileTest(p, codec, records); + } + + @Test + public void testCompressedFile() throws Exception { + // Test all the various compression codecs. + + // The following values for 'codec' should pass. + runCompressedTest(null); + runCompressedTest(CodecMap.NONE); + runCompressedTest(CodecMap.DEFLATE); + + thrown.expect(UnsupportedCodecException.class); + thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException for lzo"); + runCompressedTest(CodecMap.LZO); + } + + @Test + public void testCryptoOutputStreamClosingDoesNotThrowExceptionAndClosedProperly() throws Exception { + // Tests that closing CryptoOutputStream doesn't throw exception neither with Java 7 nor with Java 8 + // For a detailed explanation see SQOOP-3243 + CryptoOutputStream cryptoOutputStream = createCryptoOutputStream(); + FSDataOutputStream wrappedCryptoOutputStream = new FSDataOutputStream(cryptoOutputStream, null); + + Path mockPath = spy(new Path("file://" + TEMP_BASE_DIR, "binary.lob")); + FileSystem mockFileSystem = mock(FileSystem.class, withSettings().defaultAnswer(CALLS_REAL_METHODS.get())); + + doReturn(mockFileSystem).when(mockPath).getFileSystem(conf); + doReturn(null).when(mockFileSystem).getWorkingDirectory(); + doReturn(wrappedCryptoOutputStream).when(mockFileSystem).create(mockPath); + doReturn(new URI("file:///")).when(mockFileSystem).getUri(); + + LobFile.Writer writer = LobFile.create(mockPath, conf); + + writer.close(); + + verify(cryptoOutputStream).close(); + } + + + public CryptoOutputStream createCryptoOutputStream() throws Exception { + final byte[] BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + Path p = new Path(TEMP_BASE_DIR, "binary.lob"); + + FSDataOutputStream fsDataOutputStream = fs.create(p); + CryptoOutputStream cryptoOutputStream = spy(new CryptoOutputStream(fsDataOutputStream, new JceAesCtrCryptoCodec(), 512, BYTES, BYTES)); + + return cryptoOutputStream; + } + +} + http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestNamedFifo.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/io/TestNamedFifo.java b/src/test/org/apache/sqoop/io/TestNamedFifo.java new file mode 100644 index 0000000..a93784e --- /dev/null +++ b/src/test/org/apache/sqoop/io/TestNamedFifo.java @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.io; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; + +import org.apache.hadoop.util.StringUtils; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Test the named fifo utility. + */ +public class TestNamedFifo { + + public static final Log LOG = LogFactory.getLog( + TestNamedFifo.class.getName()); + + public static final Path TEMP_BASE_DIR; + + static { + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + if (!tmpDir.endsWith(File.separator)) { + tmpDir = tmpDir + File.separator; + } + + TEMP_BASE_DIR = new Path(new Path(tmpDir), "namedfifo"); + } + + private Configuration conf; + private FileSystem fs; + + @Before + public void setUp() throws Exception { + conf = new Configuration(); + conf.set("fs.default.name", "file:///"); + + fs = FileSystem.getLocal(conf); + fs.mkdirs(TEMP_BASE_DIR); + } + + static final String MSG = "THIS IS THE MESSAGE\n"; + static final String MSG2 = "Here is a follow-up.\n"; + + private static class ReaderThread extends Thread { + private File file; + private IOException exception; + + public ReaderThread(File f) { + this.file = f; + } + + /** return any exception during the run method. */ + public IOException getException() { + return this.exception; + } + + public void run() { + BufferedReader r = null; + try { + r = new BufferedReader(new InputStreamReader( + new FileInputStream(file))); + + // Assert that after a flush, we get back what we wrote. + String line = r.readLine(); + if (!MSG.trim().equals(line)) { + throw new IOException("Expected " + MSG.trim() + " but got " + + line); + } + + // Assert that after closing the writer, we get back what + // we wrote again. + line = r.readLine(); + if (null == line) { + throw new IOException("line2 was null"); + } else if (!MSG2.trim().equals(line)) { + throw new IOException("Expected " + MSG2.trim() + " but got " + + line); + } + } catch (IOException ioe) { + this.exception = ioe; + } finally { + if (null != r) { + try { + r.close(); + } catch (IOException ioe) { + LOG.warn("Error closing reader: " + ioe); + } + } + } + } + } + + private static class WriterThread extends Thread { + private File file; + private IOException exception; + + public WriterThread(File f) { + this.file = f; + } + + /** return any exception during the run method. */ + public IOException getException() { + return this.exception; + } + + public void run() { + BufferedWriter w = null; + try { + w = new BufferedWriter(new OutputStreamWriter( + new FileOutputStream(file))); + + w.write(MSG); + w.flush(); + + w.write(MSG2); + } catch (IOException ioe) { + this.exception = ioe; + } finally { + if (null != w) { + try { + w.close(); + } catch (IOException ioe) { + LOG.warn("Error closing writer: " + ioe); + } + } + } + } + } + + @Test + public void testNamedFifo() throws Exception { + + if (Shell.WINDOWS) { + // NamedFifo uses Linux specific commands like mknod + // and mkfifo, so skip the test on Windows OS + LOG.warn("Named FIFO is not supported on Windows. Skipping test"); + return; + } + + File root = new File(TEMP_BASE_DIR.toString()); + File fifo = new File(root, "foo-fifo"); + + NamedFifo nf = new NamedFifo(fifo); + nf.create(); + + File returned = nf.getFile(); + + // These should be the same object. + assertEquals(fifo, returned); + + ReaderThread rt = new ReaderThread(returned); + WriterThread wt = new WriterThread(returned); + + rt.start(); + wt.start(); + + rt.join(); + wt.join(); + + IOException rex = rt.getException(); + IOException wex = wt.getException(); + + if (null != rex) { + LOG.error("reader exception: " + StringUtils.stringifyException(rex)); + } + + if (null != wex) { + LOG.error("writer exception: " + StringUtils.stringifyException(wex)); + } + + assertNull(rex); + assertNull(wex); + } +} + http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java b/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java new file mode 100644 index 0000000..c59aa26 --- /dev/null +++ b/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java @@ -0,0 +1,318 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.io; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.File; +import java.io.FileInputStream; +import java.util.zip.GZIPInputStream; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.compress.GzipCodec; + +import org.apache.sqoop.testutil.ImportJobTestCase; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Test that the splittable buffered writer system works. + */ +public class TestSplittableBufferedWriter { + + public static final Log LOG = LogFactory.getLog( + TestSplittableBufferedWriter.class.getName()); + + private String getWriteDir() { + return new File(ImportJobTestCase.TEMP_BASE_DIR, + "bufferedWriterTest").toString(); + } + + private Path getWritePath() { + return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest"); + } + + /** Create the directory where we'll write our test files to; and + * make sure it has no files in it. + */ + private void ensureEmptyWriteDir() throws IOException { + FileSystem fs = FileSystem.getLocal(getConf()); + Path writeDir = getWritePath(); + + fs.mkdirs(writeDir); + + FileStatus [] stats = fs.listStatus(writeDir); + + for (FileStatus stat : stats) { + if (stat.isDir()) { + fail("setUp(): Write directory " + writeDir + + " contains subdirectories"); + } + + LOG.debug("setUp(): Removing " + stat.getPath()); + if (!fs.delete(stat.getPath(), false)) { + fail("setUp(): Could not delete residual file " + stat.getPath()); + } + } + + if (!fs.exists(writeDir)) { + fail("setUp: Could not create " + writeDir); + } + } + + @Before + public void setUp() throws IOException { + ensureEmptyWriteDir(); + } + + private Configuration getConf() { + Configuration conf = new Configuration(); + conf.set("fs.default.name", "file:///"); + return conf; + } + + /** Verifies contents of an InputStream. Closes the InputStream on + * its way out. Fails the test if the file doesn't match the expected set + * of lines. + */ + private void verifyFileContents(InputStream is, String [] lines) + throws IOException { + BufferedReader r = new BufferedReader(new InputStreamReader(is)); + try { + for (String expectedLine : lines) { + String actualLine = r.readLine(); + assertNotNull(actualLine); + assertEquals("Input line mismatch", expectedLine, actualLine); + } + + assertNull("Stream had additional contents after expected line", + r.readLine()); + } finally { + r.close(); + try { + is.close(); + } catch (IOException ioe) { + // ignore IOE; may be closed by reader. + } + } + } + + private void verifyFileExists(Path p) throws IOException { + FileSystem fs = FileSystem.getLocal(getConf()); + assertTrue("File not found: " + p, fs.exists(p)); + } + + private void verifyFileDoesNotExist(Path p) throws IOException { + FileSystem fs = FileSystem.getLocal(getConf()); + assertFalse("File found: " + p + " and we did not expect it", fs.exists(p)); + } + + @Test + public void testNonSplittingTextFile() throws IOException { + SplittingOutputStream os = new SplittingOutputStream(getConf(), + getWritePath(), "nonsplit-", 0, null); + try { + SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); + try { + w.allowSplit(); + w.write("This is a string!"); + w.newLine(); + w.write("This is another string!"); + w.allowSplit(); + } finally { + w.close(); + } + } finally { + try { + os.close(); + } catch (IOException ioe) { + // Ignored; may be thrown because w is already closed. + } + } + + // Ensure we made exactly one file. + Path writePath = new Path(getWritePath(), "nonsplit-00000"); + Path badPath = new Path(getWritePath(), "nonsplit-00001"); + verifyFileExists(writePath); + verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file. + + // Now ensure all the data got there. + String [] expectedLines = { + "This is a string!", + "This is another string!", + }; + + InputStream fis = new FileInputStream(new File(getWriteDir(), + "nonsplit-00000")); + try { + verifyFileContents(fis, expectedLines); + } finally { + try { + fis.close(); + } catch (IOException ioe) { + // Ignored; may be closed by verifyFileContents(). + } + } + } + + @Test + public void testNonSplittingGzipFile() throws IOException { + SplittingOutputStream os = new SplittingOutputStream(getConf(), + getWritePath(), "nonsplit-", 0, new GzipCodec()); + SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); + try { + w.allowSplit(); + w.write("This is a string!"); + w.newLine(); + w.write("This is another string!"); + w.allowSplit(); + } finally { + w.close(); + } + + // Ensure we made exactly one file. + Path writePath = new Path(getWritePath(), "nonsplit-00000.gz"); + Path badPath = new Path(getWritePath(), "nonsplit-00001.gz"); + verifyFileExists(writePath); + verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file. + + // Now ensure all the data got there. + String [] expectedLines = { + "This is a string!", + "This is another string!", + }; + verifyFileContents( + new GZIPInputStream(new FileInputStream(new File(getWriteDir(), + "nonsplit-00000.gz"))), expectedLines); + } + + @Test + public void testSplittingTextFile() throws IOException { + SplittingOutputStream os = new SplittingOutputStream(getConf(), + getWritePath(), "split-", 10, null); + try { + SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); + try { + w.allowSplit(); + w.write("This is a string!"); + w.newLine(); + w.write("This is another string!"); + } finally { + w.close(); + } + } finally { + try { + os.close(); + } catch (IOException ioe) { + // Ignored; may be thrown because w is already closed. + } + } + + // Ensure we made exactly two files. + Path writePath = new Path(getWritePath(), "split-00000"); + Path writePath2 = new Path(getWritePath(), "split-00001"); + Path badPath = new Path(getWritePath(), "split-00002"); + verifyFileExists(writePath); + verifyFileExists(writePath2); + verifyFileDoesNotExist(badPath); // Ensure we didn't make three files. + + // Now ensure all the data got there. + String [] expectedLines0 = { + "This is a string!", + }; + InputStream fis = new FileInputStream(new File(getWriteDir(), + "split-00000")); + try { + verifyFileContents(fis, expectedLines0); + } finally { + try { + fis.close(); + } catch (IOException ioe) { + // ignored; may be generated because fis closed in verifyFileContents. + } + } + + String [] expectedLines1 = { + "This is another string!", + }; + fis = new FileInputStream(new File(getWriteDir(), "split-00001")); + try { + verifyFileContents(fis, expectedLines1); + } finally { + try { + fis.close(); + } catch (IOException ioe) { + // Ignored; may be thrown because it's closed in verifyFileContents. + } + } + } + + @Test + public void testSplittingGzipFile() throws IOException { + SplittingOutputStream os = new SplittingOutputStream(getConf(), + getWritePath(), "splitz-", 3, new GzipCodec()); + SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); + try { + w.write("This is a string!"); + w.newLine(); + w.write("This is another string!"); + } finally { + w.close(); + } + + // Ensure we made exactly two files. + Path writePath = new Path(getWritePath(), "splitz-00000.gz"); + Path writePath2 = new Path(getWritePath(), "splitz-00001.gz"); + Path badPath = new Path(getWritePath(), "splitz-00002.gz"); + verifyFileExists(writePath); + verifyFileExists(writePath2); + verifyFileDoesNotExist(badPath); // Ensure we didn't make three files. + + // Now ensure all the data got there. + String [] expectedLines0 = { + "This is a string!", + }; + verifyFileContents( + new GZIPInputStream(new FileInputStream(new File(getWriteDir(), + "splitz-00000.gz"))), expectedLines0); + + String [] expectedLines1 = { + "This is another string!", + }; + verifyFileContents( + new GZIPInputStream(new FileInputStream(new File(getWriteDir(), + "splitz-00001.gz"))), expectedLines1); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestBlobRef.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/lib/TestBlobRef.java b/src/test/org/apache/sqoop/lib/TestBlobRef.java new file mode 100644 index 0000000..b271d3c --- /dev/null +++ b/src/test/org/apache/sqoop/lib/TestBlobRef.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.lib; + +import java.io.*; + +import org.apache.sqoop.testutil.BaseSqoopTestCase; +import org.apache.sqoop.testutil.CommonArgs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.sqoop.io.LobFile; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test that the BlobRef.parse() method does the right thing. + * Note that we don't support inline parsing here; we only expect this to + * really work for external BLOBs. + */ +public class TestBlobRef { + + @Test + public void testEmptyStr() { + BlobRef r = BlobRef.parse(""); + assertFalse(r.isExternal()); + } + + @Test + public void testInline() throws IOException { + BlobRef r = BlobRef.parse("foo"); + assertFalse(r.isExternal()); + } + + @Test + public void testEmptyFile() { + BlobRef r = BlobRef.parse("externalLob()"); + assertFalse(r.isExternal()); + + r = BlobRef.parse("externalLob(lf,,0,0)"); + assertTrue(r.isExternal()); + assertEquals("externalLob(lf,,0,0)", r.toString()); + } + + @Test + public void testInlineNearMatch() { + BlobRef r = BlobRef.parse("externalLob(foo)bar"); + assertFalse(r.isExternal()); + + r = BlobRef.parse("externalLob(foo)"); + assertFalse(r.isExternal()); + + r = BlobRef.parse("externalLob(lf,foo)"); + assertFalse(r.isExternal()); + + r = BlobRef.parse("externalLob(lf,foo,1,2)x"); + assertFalse(r.isExternal()); + } + + @Test + public void testExternal() throws IOException { + final byte [] DATA = { 1, 2, 3, 4, 5 }; + final String FILENAME = "blobdata"; + + doExternalTest(DATA, FILENAME); + } + + @Test + public void testExternalSubdir() throws IOException { + final byte [] DATA = { 1, 2, 3, 4, 5 }; + final String FILENAME = "_lob/blobdata"; + + try { + doExternalTest(DATA, FILENAME); + } finally { + // remove dir we made. + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + Path lobDir = new Path(new Path(tmpDir), "_lob"); + fs.delete(lobDir, true); + } + } + + private void doExternalTest(final byte [] data, final String filename) + throws IOException { + + Configuration conf = new Configuration(); + if (!BaseSqoopTestCase.isOnPhysicalCluster()) { + conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); + } + FileSystem fs = FileSystem.get(conf); + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + + Path tmpPath = new Path(tmpDir); + Path blobFile = new Path(tmpPath, filename); + + // make any necessary parent dirs. + Path blobParent = blobFile.getParent(); + if (!fs.exists(blobParent)) { + fs.mkdirs(blobParent); + } + + LobFile.Writer lw = LobFile.create(blobFile, conf, false); + try { + long off = lw.tell(); + long len = data.length; + OutputStream os = lw.writeBlobRecord(len); + os.write(data, 0, data.length); + os.close(); + lw.close(); + + String refString = "externalLob(lf," + filename + + "," + off + "," + len + ")"; + BlobRef blob = BlobRef.parse(refString); + assertTrue(blob.isExternal()); + assertEquals(refString, blob.toString()); + InputStream is = blob.getDataStream(conf, tmpPath); + assertNotNull(is); + + byte [] buf = new byte[4096]; + int bytes = is.read(buf, 0, 4096); + is.close(); + + assertEquals(data.length, bytes); + for (int i = 0; i < bytes; i++) { + assertEquals(data[i], buf[i]); + } + } finally { + fs.delete(blobFile, false); + } + } +} + http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestBooleanParser.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/lib/TestBooleanParser.java b/src/test/org/apache/sqoop/lib/TestBooleanParser.java new file mode 100644 index 0000000..914ab37 --- /dev/null +++ b/src/test/org/apache/sqoop/lib/TestBooleanParser.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.lib; + + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test the boolean parser. + */ +public class TestBooleanParser { + + @Test + public void testBoolParser() { + assertTrue(BooleanParser.valueOf("true")); + assertTrue(BooleanParser.valueOf("TRUE")); + assertTrue(BooleanParser.valueOf("True")); + assertTrue(BooleanParser.valueOf("t")); + assertTrue(BooleanParser.valueOf("T")); + assertTrue(BooleanParser.valueOf("on")); + assertTrue(BooleanParser.valueOf("On")); + assertTrue(BooleanParser.valueOf("ON")); + assertTrue(BooleanParser.valueOf("yes")); + assertTrue(BooleanParser.valueOf("yEs")); + assertTrue(BooleanParser.valueOf("YES")); + assertTrue(BooleanParser.valueOf("1")); + + assertFalse(BooleanParser.valueOf(null)); + + assertFalse(BooleanParser.valueOf("no")); + assertFalse(BooleanParser.valueOf("false")); + assertFalse(BooleanParser.valueOf("FALSE")); + assertFalse(BooleanParser.valueOf("0")); + assertFalse(BooleanParser.valueOf("off")); + assertFalse(BooleanParser.valueOf("OFF")); + assertFalse(BooleanParser.valueOf("anything else in the world")); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestClobRef.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/lib/TestClobRef.java b/src/test/org/apache/sqoop/lib/TestClobRef.java new file mode 100644 index 0000000..f94d1a8 --- /dev/null +++ b/src/test/org/apache/sqoop/lib/TestClobRef.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.lib; + +import java.io.*; + +import org.apache.sqoop.testutil.BaseSqoopTestCase; +import org.apache.sqoop.testutil.CommonArgs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.sqoop.io.LobFile; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test parsing of ClobRef objects. + */ +public class TestClobRef { + + @Test + public void testEmptyStr() { + ClobRef r = ClobRef.parse(""); + assertFalse(r.isExternal()); + assertEquals("", r.toString()); + } + + @Test + public void testInline() throws IOException { + ClobRef r = ClobRef.parse("foo"); + assertFalse(r.isExternal()); + assertEquals("foo", r.toString()); + + Reader reader = r.getDataStream(null, null); + assertNotNull(reader); + char [] buf = new char[4096]; + int chars = reader.read(buf, 0, 4096); + reader.close(); + + String str = new String(buf, 0, chars); + assertEquals("foo", str); + } + + @Test + public void testEmptyFile() { + ClobRef r = ClobRef.parse("externalLob()"); + assertFalse(r.isExternal()); + assertEquals("externalLob()", r.toString()); + + r = ClobRef.parse("externalLob(lf,,0,0)"); + assertTrue(r.isExternal()); + assertEquals("externalLob(lf,,0,0)", r.toString()); + } + + @Test + public void testInlineNearMatch() { + ClobRef r = ClobRef.parse("externalLob(foo)bar"); + assertFalse(r.isExternal()); + assertEquals("externalLob(foo)bar", r.toString()); + + r = ClobRef.parse("externalLob(foo)"); + assertFalse(r.isExternal()); + assertEquals("externalLob(foo)", r.getData()); + + r = ClobRef.parse("externalLob(lf,foo)"); + assertFalse(r.isExternal()); + assertEquals("externalLob(lf,foo)", r.getData()); + + r = ClobRef.parse("externalLob(lf,foo,1,2)x"); + assertFalse(r.isExternal()); + assertEquals("externalLob(lf,foo,1,2)x", r.getData()); + } + + @Test + public void testExternal() throws IOException { + final String DATA = "This is the clob data!"; + final String FILENAME = "clobdata"; + + doExternalTest(DATA, FILENAME); + } + + @Test + public void testExternalSubdir() throws IOException { + final String DATA = "This is the clob data!"; + final String FILENAME = "_lob/clobdata"; + + try { + doExternalTest(DATA, FILENAME); + } finally { + // remove dir we made. + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.getLocal(conf); + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + Path lobDir = new Path(new Path(tmpDir), "_lob"); + fs.delete(lobDir, true); + } + } + + private void doExternalTest(final String data, final String filename) + throws IOException { + + Configuration conf = new Configuration(); + if (!BaseSqoopTestCase.isOnPhysicalCluster()) { + conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); + } + FileSystem fs = FileSystem.get(conf); + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + + Path tmpPath = new Path(tmpDir); + Path clobFile = new Path(tmpPath, filename); + + // make any necessary parent dirs. + Path clobParent = clobFile.getParent(); + if (!fs.exists(clobParent)) { + fs.mkdirs(clobParent); + } + + LobFile.Writer lw = LobFile.create(clobFile, conf, true); + try { + long off = lw.tell(); + long len = data.length(); + Writer w = lw.writeClobRecord(len); + w.append(data); + w.close(); + lw.close(); + + String refString = "externalLob(lf," + filename + + "," + off + "," + len + ")"; + ClobRef clob = ClobRef.parse(refString); + assertTrue(clob.isExternal()); + assertEquals(refString, clob.toString()); + Reader r = clob.getDataStream(conf, tmpPath); + assertNotNull(r); + + char [] buf = new char[4096]; + int chars = r.read(buf, 0, 4096); + r.close(); + + String str = new String(buf, 0, chars); + assertEquals(data, str); + } finally { + fs.delete(clobFile, false); + } + } +} + http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestFieldFormatter.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/lib/TestFieldFormatter.java b/src/test/org/apache/sqoop/lib/TestFieldFormatter.java new file mode 100644 index 0000000..9ac55e7 --- /dev/null +++ b/src/test/org/apache/sqoop/lib/TestFieldFormatter.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.lib; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + + +/** + * Test that the field formatter works in a variety of configurations. + */ +public class TestFieldFormatter { + + @Test + public void testAllEmpty() { + String result = FieldFormatter.escapeAndEnclose("", + new DelimiterSet(DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR, + DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR, false)); + assertEquals("", result); + } + + @Test + public void testNullArgs() { + assertNull(FieldFormatter.escapeAndEnclose(null, + new DelimiterSet('\"', DelimiterSet.NULL_CHAR, '\"', '\\', false))); + } + + @Test + public void testBasicStr() { + String result = FieldFormatter.escapeAndEnclose("foo", + DelimiterSet.DEFAULT_DELIMITERS); + assertEquals("foo", result); + } + + @Test + public void testEscapeSlash() { + String result = FieldFormatter.escapeAndEnclose("foo\\bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("foo\\\\bar", result); + } + + @Test + public void testMustEnclose() { + String result = FieldFormatter.escapeAndEnclose("foo", + new DelimiterSet(',', '\n', '\"', DelimiterSet.NULL_CHAR, true)); + assertEquals("\"foo\"", result); + } + + @Test + public void testEncloseComma1() { + String result = FieldFormatter.escapeAndEnclose("foo,bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("\"foo,bar\"", result); + } + + @Test + public void testEncloseComma2() { + String result = FieldFormatter.escapeAndEnclose("foo,bar", + new DelimiterSet(',', ',', '\"', '\\', false)); + assertEquals("\"foo,bar\"", result); + } + + @Test + public void testNoNeedToEnclose() { + String result = FieldFormatter.escapeAndEnclose( + "just another string", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("just another string", result); + } + + @Test + public void testCannotEnclose() { + // Can't enclose because encloser is nul. + // This should escape the comma instead. + String result = FieldFormatter.escapeAndEnclose("foo,bar", + new DelimiterSet(',', '\n', DelimiterSet.NULL_CHAR, '\\', false)); + + assertEquals("foo\\,bar", result); + } + + @Test + public void testEmptyCharToEscapeString() { + // test what happens when the escape char is null. It should encode the + // null char. + + char nul = DelimiterSet.NULL_CHAR; + String s = "" + nul; + assertEquals("\000", s); + } + + @Test + public void testEscapeCentralQuote() { + String result = FieldFormatter.escapeAndEnclose("foo\"bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("foo\\\"bar", result); + } + + @Test + public void testEscapeMultiCentralQuote() { + String result = FieldFormatter.escapeAndEnclose("foo\"\"bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("foo\\\"\\\"bar", result); + } + + @Test + public void testDoubleEscape() { + String result = FieldFormatter.escapeAndEnclose("foo\\\"bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("foo\\\\\\\"bar", result); + } + + @Test + public void testReverseEscape() { + String result = FieldFormatter.escapeAndEnclose("foo\"\\bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("foo\\\"\\\\bar", result); + } + + @Test + public void testQuotedEncloser() { + String result = FieldFormatter.escapeAndEnclose("foo\",bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("\"foo\\\",bar\"", result); + } + + @Test + public void testQuotedEscape() { + String result = FieldFormatter.escapeAndEnclose("foo\\,bar", + new DelimiterSet(',', '\n', '\"', '\\', false)); + assertEquals("\"foo\\\\,bar\"", result); + } +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java b/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java new file mode 100644 index 0000000..1e07d71 --- /dev/null +++ b/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.lib; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.apache.sqoop.testutil.BaseSqoopTestCase; +import org.apache.sqoop.testutil.CommonArgs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.sqoop.testutil.MockResultSet; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test deserialization of ClobRef and BlobRef fields. + */ +public class TestLargeObjectLoader { + + protected Configuration conf; + protected LargeObjectLoader loader; + protected Path outDir; + + @Before + public void setUp() throws IOException, InterruptedException { + conf = new Configuration(); + if (!BaseSqoopTestCase.isOnPhysicalCluster()) { + conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); + } + String tmpDir = System.getProperty("test.build.data", "/tmp/"); + this.outDir = new Path(System.getProperty("java.io.tmpdir")); + FileSystem fs = FileSystem.get(conf); + if (fs.exists(outDir)) { + fs.delete(outDir, true); + } + fs.mkdirs(outDir); + + loader = new LargeObjectLoader(conf, outDir); + } + + @Test + public void testReadClobRef() + throws IOException, InterruptedException, SQLException { + // This should give us an inline CLOB. + ResultSet resultSet = new MockResultSet(); + ClobRef clob = loader.readClobRef(0, resultSet); + assertNotNull(clob); + assertFalse(clob.isExternal()); + assertEquals(MockResultSet.CLOB_DATA, clob.toString()); + + // LOBs bigger than 4 bytes are now external. + conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4); + clob = loader.readClobRef(0, resultSet); + assertNotNull(clob); + assertTrue(clob.isExternal()); + loader.close(); + Reader r = clob.getDataStream(conf, outDir); + char [] buf = new char[4096]; + int chars = r.read(buf, 0, 4096); + r.close(); + String str = new String(buf, 0, chars); + assertEquals(MockResultSet.CLOB_DATA, str); + } + + @Test + public void testReadBlobRef() + throws IOException, InterruptedException, SQLException { + // This should give us an inline BLOB. + ResultSet resultSet = new MockResultSet(); + BlobRef blob = loader.readBlobRef(0, resultSet); + assertNotNull(blob); + assertFalse(blob.isExternal()); + byte [] data = blob.getData(); + byte [] blobData = MockResultSet.blobData(); + assertEquals(blobData.length, data.length); + for (int i = 0; i < data.length; i++) { + assertEquals(blobData[i], data[i]); + } + + // LOBs bigger than 4 bytes are now external. + conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4); + blob = loader.readBlobRef(0, resultSet); + assertNotNull(blob); + assertTrue(blob.isExternal()); + loader.close(); + InputStream is = blob.getDataStream(conf, outDir); + byte [] buf = new byte[4096]; + int bytes = is.read(buf, 0, 4096); + is.close(); + + assertEquals(blobData.length, bytes); + for (int i = 0; i < bytes; i++) { + assertEquals(blobData[i], buf[i]); + } + } +} + http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestRecordParser.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/lib/TestRecordParser.java b/src/test/org/apache/sqoop/lib/TestRecordParser.java new file mode 100644 index 0000000..d6844c1 --- /dev/null +++ b/src/test/org/apache/sqoop/lib/TestRecordParser.java @@ -0,0 +1,450 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.sqoop.lib; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.junit.Assert.fail; + + +/** + * Test that the record parser works in a variety of configurations. + */ +public class TestRecordParser { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private void assertListsEqual(String msg, List<String> expected, + List<String> actual) { + if (expected == null && actual != null) { + if (null == msg) { + msg = "expected null list"; + } + + fail(msg); + } else if (expected != null && actual == null) { + if (null == msg) { + msg = "expected non-null list"; + } + + fail(msg); + } else if (expected == null && actual == null) { + return; // ok. Both null; nothing to do. + } + + assert(null != expected); + assert(null != actual); + + int expectedLen = expected.size(); + int actualLen = actual.size(); + + if (expectedLen != actualLen) { + if (null == msg) { + msg = "Expected list of length " + expectedLen + + "; got " + actualLen; + } + + fail(msg); + } + + // Check the list contents. + for (int i = 0; i < expectedLen; i++) { + String expectedElem = expected.get(i); + String actualElem = actual.get(i); + + if (expectedElem == null) { + if (actualElem != null) { + if (null == msg) { + msg = "Expected null element at position " + i + + "; got [" + actualElem + "]"; + } + + fail(msg); + } + } else if (!expectedElem.equals(actualElem)) { + if (null == msg) { + msg = "Expected [" + expectedElem + "] at position " + i + + "; got [" + actualElem + "]"; + } + + fail(msg); + } + } + } + + private List<String> list(String [] items) { + + if (null == items) { + return null; + } + + ArrayList<String> asList = new ArrayList<String>(); + for (int i = 0; i < items.length; i++) { + asList.add(items[i]); + } + + return asList; + } + + @Test + public void testEmptyLine() throws RecordParser.ParseError { + // an empty line should return no fields. + + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { }; + assertListsEqual(null, list(strings), parser.parseRecord("")); + } + + @Test + public void testJustEOR() throws RecordParser.ParseError { + // a line with just a newline char should return a single zero-length field. + + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "" }; + assertListsEqual(null, list(strings), parser.parseRecord("\n")); + } + + @Test + public void testOneField() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the field" }; + assertListsEqual(null, list(strings), parser.parseRecord("the field")); + } + + @Test + public void testOneField2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the field" }; + assertListsEqual(null, list(strings), parser.parseRecord("the field\n")); + } + + @Test + public void testQuotedField1() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the field" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the field\"\n")); + } + + @Test + public void testQuotedField2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the field" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the field\"")); + } + + @Test + public void testQuotedField3() throws RecordParser.ParseError { + // quoted containing EOF + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the ,field" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the ,field\"")); + } + + @Test + public void testQuotedField4() throws RecordParser.ParseError { + // quoted containing multiple EOFs + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the ,,field" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the ,,field\"")); + } + + @Test + public void testQuotedField5() throws RecordParser.ParseError { + // quoted containing EOF and EOR + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the ,\nfield" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the ,\nfield\"")); + } + + @Test + public void testQuotedField6() throws RecordParser.ParseError { + // quoted containing EOR + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the \nfield" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the \nfield\"")); + } + + @Test + public void testQuotedField7() throws RecordParser.ParseError { + // quoted containing multiple EORs + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the \n\nfield" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the \n\nfield\"")); + } + + @Test + public void testQuotedField8() throws RecordParser.ParseError { + // quoted containing escaped quoted char + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the \"field" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"the \\\"field\"")); + } + + @Test + public void testUnquotedEscape1() throws RecordParser.ParseError { + // field without quotes with an escaped EOF char. + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the ,field" }; + assertListsEqual(null, list(strings), parser.parseRecord("the \\,field")); + } + + @Test + public void testUnquotedEscape2() throws RecordParser.ParseError { + // field without quotes with an escaped escape char. + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "the \\field" }; + assertListsEqual(null, list(strings), parser.parseRecord("the \\\\field")); + } + + @Test + public void testTwoFields1() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), parser.parseRecord("field1,field2")); + } + + @Test + public void testTwoFields2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), + parser.parseRecord("field1,field2\n")); + } + + @Test + public void testTwoFields3() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"field1\",field2\n")); + } + + @Test + public void testTwoFields4() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), + parser.parseRecord("field1,\"field2\"\n")); + } + + @Test + public void testTwoFields5() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), + parser.parseRecord("field1,\"field2\"")); + } + + @Test + public void testRequiredQuotes0() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', true)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"field1\",\"field2\"\n")); + } + + @Test + public void testRequiredQuotes1() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', true)); + String [] strings = { "field1", "field2" }; + assertListsEqual(null, list(strings), + parser.parseRecord("\"field1\",\"field2\"")); + } + + @Test + public void testRequiredQuotes2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', true)); + + thrown.expect(RecordParser.ParseError.class); + thrown.reportMissingExceptionWithMessage("Expected parse error for required quotes"); + parser.parseRecord("\"field1\",field2"); + } + + @Test + public void testRequiredQuotes3() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', true)); + + thrown.expect(RecordParser.ParseError.class); + thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes"); + parser.parseRecord("field1,\"field2\""); + } + + @Test + public void testRequiredQuotes4() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', true)); + + thrown.expect(RecordParser.ParseError.class); + thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes"); + parser.parseRecord("field1,\"field2\"\n"); + } + + @Test + public void testNull() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', true)); + String input = null; + + thrown.expect(RecordParser.ParseError.class); + thrown.reportMissingExceptionWithMessage("Expected ParseError for null string"); + parser.parseRecord(input); + } + + + @Test + public void testEmptyFields1() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "", ""}; + assertListsEqual(null, list(strings), parser.parseRecord(",")); + } + + @Test + public void testEmptyFields2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "", "" }; + assertListsEqual(null, list(strings), parser.parseRecord(",\n")); + } + + @Test + public void testEmptyFields3() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "", "", "" }; + assertListsEqual(null, list(strings), parser.parseRecord(",,\n")); + } + + @Test + public void testEmptyFields4() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "", "foo", "" }; + assertListsEqual(null, list(strings), parser.parseRecord(",foo,\n")); + } + + @Test + public void testEmptyFields5() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "", "foo", "" }; + assertListsEqual(null, list(strings), parser.parseRecord(",foo,")); + } + + @Test + public void testEmptyFields6() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "foo", "" }; + assertListsEqual(null, list(strings), parser.parseRecord("foo,")); + } + + @Test + public void testTrailingText() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "foo", "bar" }; + assertListsEqual(null, list(strings), parser.parseRecord("foo,bar\nbaz")); + } + + @Test + public void testTrailingText2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "" }; + assertListsEqual(null, list(strings), parser.parseRecord("\nbaz")); + } + + @Test + public void testLeadingEscape() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', '\n', '\"', '\\', false)); + String [] strings = { "\nbaz" }; + assertListsEqual(null, list(strings), parser.parseRecord("\\\nbaz")); + } + + @Test + public void testEofIsEor() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', ',', '\"', '\\', false)); + String [] strings = { "three", "different", "fields" }; + assertListsEqual(null, list(strings), + parser.parseRecord("three,different,fields")); + } + + @Test + public void testEofIsEor2() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', ',', '\"', '\\', false)); + String [] strings = { "three", "different", "fields" }; + assertListsEqual(null, list(strings), + parser.parseRecord("three,\"different\",fields")); + } + + @Test + public void testRepeatedParse() throws RecordParser.ParseError { + RecordParser parser = new RecordParser( + new DelimiterSet(',', ',', '\"', '\\', false)); + String [] strings = { "three", "different", "fields" }; + assertListsEqual(null, list(strings), + parser.parseRecord("three,\"different\",fields")); + + String [] strings2 = { "foo", "bar" }; + assertListsEqual(null, list(strings2), + parser.parseRecord("foo,\"bar\"")); + } + +} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java b/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java index fd72ef4..8e16324 100644 --- a/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java +++ b/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java @@ -18,8 +18,8 @@ package org.apache.sqoop.manager; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.metastore.JobData; +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.metastore.JobData; import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.conf.Configuration; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/TestMainframeManager.java ---------------------------------------------------------------------- diff --git a/src/test/org/apache/sqoop/manager/TestMainframeManager.java b/src/test/org/apache/sqoop/manager/TestMainframeManager.java index 9359ac4..97e48e8 100644 --- a/src/test/org/apache/sqoop/manager/TestMainframeManager.java +++ b/src/test/org/apache/sqoop/manager/TestMainframeManager.java @@ -36,13 +36,11 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.cloudera.sqoop.ConnFactory; -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.manager.ConnManager; -import com.cloudera.sqoop.manager.ImportJobContext; -import com.cloudera.sqoop.metastore.JobData; -import com.cloudera.sqoop.testutil.BaseSqoopTestCase; -import com.cloudera.sqoop.util.ImportException; +import org.apache.sqoop.ConnFactory; +import org.apache.sqoop.SqoopOptions; +import org.apache.sqoop.metastore.JobData; +import org.apache.sqoop.testutil.BaseSqoopTestCase; +import org.apache.sqoop.util.ImportException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull;
