http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/io/TestLobFile.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/io/TestLobFile.java b/src/test/com/cloudera/sqoop/io/TestLobFile.java deleted file mode 100644 index ac461fd..0000000 --- a/src/test/com/cloudera/sqoop/io/TestLobFile.java +++ /dev/null @@ -1,642 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.io; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Reader; -import java.io.Writer; -import java.net.URI; -import java.nio.CharBuffer; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.crypto.CryptoOutputStream; -import org.apache.hadoop.crypto.JceAesCtrCryptoCodec; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.junit.Before; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Answers.CALLS_REAL_METHODS; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.withSettings; - -import org.junit.Rule; -import org.junit.rules.ExpectedException; - -/** - * Test the LobFile reader/writer implementation. - */ -public class TestLobFile { - - public static final Log LOG = LogFactory.getLog( - TestLobFile.class.getName()); - - public static final Path TEMP_BASE_DIR; - - static { - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - if (!tmpDir.endsWith(File.separator)) { - tmpDir = tmpDir + File.separator; - } - - TEMP_BASE_DIR = new Path(new Path(tmpDir), "lobtest"); - } - - private Configuration conf; - private FileSystem fs; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Before - public void setUp() throws Exception { - conf = new Configuration(); - conf.set("fs.default.name", "file:///"); - - fs = FileSystem.getLocal(conf); - fs.mkdirs(TEMP_BASE_DIR); - } - - private long[] writeClobFile(Path p, String codec, - String... records) throws Exception { - if (fs.exists(p)) { - fs.delete(p, false); - } - - // memorize the offsets of each record we write. - long[] offsets = new long[records.length]; - - // Create files with four entries per index segment. - LobFile.Writer writer = LobFile.create(p, conf, true, codec, 4); - - int i = 0; - for (String r : records) { - offsets[i++] = writer.tell(); - Writer w = writer.writeClobRecord(r.length()); - w.write(r); - w.close(); - } - - writer.close(); - return offsets; - } - - private void verifyClobFile(Path p, String... expectedRecords) - throws Exception { - - LobFile.Reader reader = LobFile.open(p, conf); - - int recNum = 0; - - while (reader.next()) { - // We should have a record of the same length as the expected one. - String expected = expectedRecords[recNum]; - assertTrue(reader.isRecordAvailable()); - assertEquals(expected.length(), reader.getRecordLen()); - Reader r = reader.readClobRecord(); - - // Read in the record and assert that we got enough characters out. - CharBuffer buf = CharBuffer.allocate(expected.length()); - int bytesRead = 0; - while (bytesRead < expected.length()) { - int thisRead = r.read(buf); - LOG.info("performed read of " + thisRead + " chars"); - if (-1 == thisRead) { - break; - } - - bytesRead += thisRead; - } - - LOG.info("Got record of " + bytesRead + " chars"); - assertEquals(expected.length(), bytesRead); - char[] charData = buf.array(); - String finalRecord = new String(charData); - assertEquals(expected, finalRecord); - - recNum++; - } - - // Check that we got everything. - assertEquals(expectedRecords.length, recNum); - - reader.close(); - - thrown.expect(IOException.class); - thrown.reportMissingExceptionWithMessage("Expected IOException calling next after close"); - reader.next(); - - // A second close shouldn't hurt anything. This should be a no-op. - reader.close(); - } - - private void runClobFileTest(Path p, String codec, - String... records) throws Exception { - writeClobFile(p, codec, records); - verifyClobFile(p, records); - fs.delete(p, false); - } - - @Test - public void testEmptyRecord() throws Exception { - runClobFileTest(new Path(TEMP_BASE_DIR, "empty.lob"), null); - } - - @Test - public void testSingleRecord() throws Exception { - runClobFileTest(new Path(TEMP_BASE_DIR, "single.lob"), - null, "this is a single record!"); - } - - @Test - public void testMultiRecords() throws Exception { - runClobFileTest(new Path(TEMP_BASE_DIR, "multi.lob"), - CodecMap.NONE, - "this is the first record", - "this is the second record. I assure you that this record is long.", - "yet one more record graces this file."); - } - - @Test - public void testMultiIndexSegments() throws Exception { - // Test that we can use multiple IndexSegments. - runClobFileTest(new Path(TEMP_BASE_DIR, "multi-index.lob"), - null, - "this is the first record", - "this is the second record. I assure you that this record is long.", - "record number three", - "last one in first index segment", - "first in the second index segment", - "yet one more record graces this file."); - } - - /** - * Run a test where we read only a fraction of the first record, - * but then read the second record completely. Verify that we - * can re-align on a record boundary correctly. This test requires - * at least 3 records. - * - * @param p the path to the file to create. - * @param firstLine the first line of the first reord - * @param records All of the records to write to the file. - */ - private void runLineAndRecordTest(Path p, String firstLine, - String... records) throws Exception { - - assertTrue("This test requires 3+ records", records.length > 2); - - writeClobFile(p, null, records); - - LobFile.Reader reader = LobFile.open(p, conf); - - // We should not yet be aligned. - assertFalse(reader.isRecordAvailable()); - assertTrue(reader.next()); - // Now we should be. - assertTrue(reader.isRecordAvailable()); - - // Read just one line from the record. - Reader r = reader.readClobRecord(); - BufferedReader br = new BufferedReader(r); - String line = br.readLine(); - assertEquals(firstLine, line); - - br.close(); - r.close(); - - // We should no longer be aligned on a record start. - assertFalse(reader.isRecordAvailable()); - - // We should now be able to get to record two. - assertTrue(reader.next()); - - // This should be nicely aligned even if the first record was not - // completely consumed by a client. - r = reader.readClobRecord(); - CharBuffer buf = CharBuffer.allocate(records[1].length()); - r.read(buf); - r.close(); - char[] chars = buf.array(); - String s = new String(chars); - assertEquals(records[1], s); - - // Close the reader before we consume the entire file. - reader.close(); - assertFalse(reader.isRecordAvailable()); - } - - @Test - public void testVeryShortRead() throws Exception { - // Read only a small fraction of a record, ensure that we can - // read the next record, even when we've left more than a 16-byte - // quantity in the readahead buffer. - - Path p = new Path(TEMP_BASE_DIR, "shortread.lob"); - final String FIRST_LINE = "line1"; - final String SECOND_LINE = - "This contains much more in the record than just one line."; - final String RECORD2 = "here is the second record."; - final String RECORD3 = "The 3rd record, which we won't actually read."; - - runLineAndRecordTest(p, FIRST_LINE, - FIRST_LINE + "\n" + SECOND_LINE, - RECORD2, - RECORD3); - - } - - @Test - public void testIncompleteOverread() throws Exception { - // Read most of the first record so that we partially consume the - // next record start mark; make sure we realign properly. - - Path p = new Path(TEMP_BASE_DIR, "longread.lob"); - final String FIRST_LINE = "this is a really long line of text to read!"; - final String SECOND_LINE = "not this."; - final String RECORD2 = "Here is yet another record to verify."; - final String RECORD3 = "Nobody cares about record 3."; - - runLineAndRecordTest(p, FIRST_LINE, - FIRST_LINE + "\n" + SECOND_LINE, - RECORD2, - RECORD3); - } - - @Test - public void testSeekToRecord() throws Exception { - // Seek past the first two records and read the third. - - Path p = new Path(TEMP_BASE_DIR, "seek.lob"); - String[] records = { - "this is the first record!", - "here comes record number two. It is a bit longer.", - "this is the third record. we can read it.", - }; - - // Write the file and memorize when the third record starts. - LobFile.Writer writer = LobFile.create(p, conf, true); - - int recNum = 0; - long rec3Start = 0; - for (String r : records) { - Writer w = writer.writeClobRecord(r.length()); - w.write(r); - w.close(); - writer.finishRecord(); - if (recNum == 1) { - rec3Start = writer.tell(); - LOG.info("Record three start: " + rec3Start); - } - recNum++; - } - - writer.close(); - - // Now reopen the file for read, seek to the third record, and get it. - LobFile.Reader reader = LobFile.open(p, conf); - reader.seek(rec3Start); - assertTrue(reader.next()); - assertTrue(reader.isRecordAvailable()); - assertEquals(rec3Start, reader.getRecordOffset()); - - Reader r = reader.readClobRecord(); - CharBuffer buf = CharBuffer.allocate(records[2].length()); - r.read(buf); - r.close(); - char[] chars = buf.array(); - String s = new String(chars); - assertEquals(records[2], s); - - r.close(); - reader.close(); - } - - - /** - * Verifies that the next record in the LobFile is the expected one. - */ - private void verifyNextRecord(LobFile.Reader reader, long expectedId, - String expectedRecord) throws Exception { - assertTrue(reader.next()); - assertTrue(reader.isRecordAvailable()); - assertEquals(expectedId, reader.getRecordId()); - - Reader r = reader.readClobRecord(); - CharBuffer buf = CharBuffer.allocate(expectedRecord.length()); - int bytesRead = 0; - while (bytesRead < expectedRecord.length()) { - int thisRead = r.read(buf); - if (-1 == thisRead) { - break; - } - - bytesRead += thisRead; - } - - LOG.info("Got record of " + bytesRead + " chars"); - assertEquals(expectedRecord.length(), bytesRead); - - char[] charData = buf.array(); - String finalRecord = new String(charData); - assertEquals(expectedRecord, finalRecord); - } - - @Test - public void testManySeeks() throws Exception { - // Test that we can do gymnastics with seeking between records. - - Path p = new Path(TEMP_BASE_DIR, "manyseeks.lob"); - - String[] records = { - "first record", - "second record", - "the third record", - "rec4 is the last in IndexSeg 0", - "rec5 is first in IndexSeg 1", - "rec6 is yet another record", - "rec7 is starting to feel boring", - "rec8 is at the end of seg 1", - "rec9 is all by itself in seg 2", - }; - - // Write the records to a file, save their offsets. - long[] offsets = writeClobFile(p, null, records); - - // Sanity check that we can stream the file. - verifyClobFile(p, records); - - // Open a handle to the file. - LobFile.Reader reader = LobFile.open(p, conf); - - // Seeking to offset 0 should return the first record. - reader.seek(0); - verifyNextRecord(reader, 0, records[0]); - - // Seek to the last item in the first IndexSegment. - reader.seek(offsets[3]); - verifyNextRecord(reader, 3, records[3]); - - // Seek to just ahead of that same record. - reader.seek(offsets[3] - 10); - verifyNextRecord(reader, 3, records[3]); - - // Seek (backwards) to the first record. - reader.seek(offsets[0]); - verifyNextRecord(reader, 0, records[0]); - - // Seek to first record in second IndexSegment. - reader.seek(offsets[4]); - verifyNextRecord(reader, 4, records[4]); - - // Move backwards. - reader.seek(0); - - // Seek to "no man's land" between last offset in first IndexSeg - // and the first offset in second IndexSegment. Result should be - // the first record in second InexSegment. - reader.seek(offsets[4] - 10); - verifyNextRecord(reader, 4, records[4]); - - // Seek to past the last record. No record should be returned. - reader.seek(offsets[8] + 4); - assertFalse("Found a record past last record start.", reader.next()); - - // Seek to somewhere in the middle of IndexSegment 0. - // This should recover just fine. - reader.seek(offsets[2]); - verifyNextRecord(reader, 2, records[2]); - - // Seek to last record in IndexSegment 1. - reader.seek(offsets[3] - 1); - verifyNextRecord(reader, 3, records[3]); - - // And make sure that iteration picks up naturally from there. - verifyNextRecord(reader, 4, records[4]); - - // Seek well past the end of the file. No record should be returned. - reader.seek(50000000); - assertFalse("Found a record past expected end-of-file", reader.next()); - - // Seek to somewhere in the index. - reader.seek(offsets[8] + 32); - assertFalse("Found a record past beginning of index", reader.next()); - - // Seek to the last record (exact hit). This is a singleton IndexSegment. - reader.seek(offsets[8]); - verifyNextRecord(reader, 8, records[8]); - - // Seek to no-man's-land ahead of last record. - reader.seek(offsets[8] - 3); - verifyNextRecord(reader, 8, records[8]); - - reader.close(); - } - - /** - * Verifies that a record to be read from a lob file has - * as many bytes as we expect, and that the bytes are what we - * expect them to be. Assumes that the bytes are such that - * input[i] == i + offset. - * - * @param reader the LobFile reader to consume data from - * @param expectedDeclaredLen the size we expect the LobFile to declare - * its record length as. - * @param expectedActualLen the true number of bytes we expect to read in - * the record. - * @param offset the offset amount for each of the elements of the array. - */ - private void verifyBlobRecord(LobFile.Reader reader, - long expectedDeclaredLen, long expectedActualLen, - int offset) throws Exception { - - assertTrue(reader.next()); - assertTrue(reader.isRecordAvailable()); - assertEquals(expectedDeclaredLen, reader.getRecordLen()); - - InputStream is = reader.readBlobRecord(); - - byte[] bytes = new byte[(int) expectedActualLen]; - int numRead = is.read(bytes); - assertEquals(expectedActualLen, numRead); - - for (int i = 0; i < numRead; i++) { - assertEquals(i + offset, (int) bytes[i]); - } - - is.close(); - } - - /** - * Write a binary record to a LobFile. This allows the declared length - * of the record to disagree with the actual length (the actual length - * should be >= the declared length). - * The record written will have values v[i] = i + offset. - * - * @param writer the LobFile writer to put the record into - * @param declaredLen the length value written into the file itself - * @param actualLen the true number of bytes to write - * @param offset an amount to adjust each record's byte values by. - */ - private void writeBlobRecord(LobFile.Writer writer, long declaredLen, - long actualLen, int offset) throws Exception { - OutputStream os = writer.writeBlobRecord(declaredLen); - for (int i = 0; i < actualLen; i++) { - os.write(i + offset); - } - - os.close(); - writer.finishRecord(); - } - - /** - * Verifies a number of records that all have the same declared - * and actual record lengths. - * - * @param p the path to the LobFile to open - * @param numRecords the number of records to expect - * @param declaredLen the declared length of each record in the file - * @param actualLen the true number of bytes we expect to read per record. - */ - private void verifyBlobRecords(Path p, int numRecords, - long declaredLen, long actualLen) throws Exception { - - LobFile.Reader reader = LobFile.open(p, conf); - for (int i = 0; i < numRecords; i++) { - verifyBlobRecord(reader, declaredLen, actualLen, i); - } - assertFalse(reader.next()); - reader.close(); - } - - @Test - public void testBinaryRecords() throws Exception { - // Write a BLOB file and read it all back. - - final long RECORD_LEN = 32; - final int NUM_RECORDS = 2; - Path p = new Path(TEMP_BASE_DIR, "binary.lob"); - LobFile.Writer writer = LobFile.create(p, conf); - - for (int i = 0; i < NUM_RECORDS; i++) { - writeBlobRecord(writer, RECORD_LEN, RECORD_LEN, i); - } - - writer.close(); - - // Now check the read-back on those records. - verifyBlobRecords(p, NUM_RECORDS, RECORD_LEN, RECORD_LEN); - } - - @Test - public void testOverLengthBinaryRecord() throws Exception { - // Write a record with a declared length shorter than the - // actual length, and read it back. - - final long ACTUAL_RECORD_LEN = 48; - final long DECLARED_RECORD_LEN = 32; - final int NUM_RECORDS = 2; - - Path p = new Path(TEMP_BASE_DIR, "overlength.lob"); - LobFile.Writer writer = LobFile.create(p, conf); - - for (int i = 0; i < NUM_RECORDS; i++) { - writeBlobRecord(writer, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN, i); - } - - writer.close(); - - // Now read them back. - verifyBlobRecords(p, NUM_RECORDS, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN); - } - - private void runCompressedTest(String codec) throws Exception { - LOG.info("Testing with codec: " + codec); - Path p = new Path(TEMP_BASE_DIR, "compressed-" + codec + ".lob"); - String[] records = { - "this is the first record, It should be compressed a lot!", - "record 2 record 2 record 2 record 2 2 2 2 2 2 2 2 2 2 2 2", - "and a third and a third yes this is the third", - }; - - runClobFileTest(p, codec, records); - } - - @Test - public void testCompressedFile() throws Exception { - // Test all the various compression codecs. - - // The following values for 'codec' should pass. - runCompressedTest(null); - runCompressedTest(CodecMap.NONE); - runCompressedTest(CodecMap.DEFLATE); - - thrown.expect(UnsupportedCodecException.class); - thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException for lzo"); - runCompressedTest(CodecMap.LZO); - } - - @Test - public void testCryptoOutputStreamClosingDoesNotThrowExceptionAndClosedProperly() throws Exception { - // Tests that closing CryptoOutputStream doesn't throw exception neither with Java 7 nor with Java 8 - // For a detailed explanation see SQOOP-3243 - CryptoOutputStream cryptoOutputStream = createCryptoOutputStream(); - FSDataOutputStream wrappedCryptoOutputStream = new FSDataOutputStream(cryptoOutputStream, null); - - Path mockPath = spy(new Path("file://" + TEMP_BASE_DIR, "binary.lob")); - FileSystem mockFileSystem = mock(FileSystem.class, withSettings().defaultAnswer(CALLS_REAL_METHODS.get())); - - doReturn(mockFileSystem).when(mockPath).getFileSystem(conf); - doReturn(null).when(mockFileSystem).getWorkingDirectory(); - doReturn(wrappedCryptoOutputStream).when(mockFileSystem).create(mockPath); - doReturn(new URI("file:///")).when(mockFileSystem).getUri(); - - LobFile.Writer writer = LobFile.create(mockPath, conf); - - writer.close(); - - verify(cryptoOutputStream).close(); - } - - - public CryptoOutputStream createCryptoOutputStream() throws Exception { - final byte[] BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; - Path p = new Path(TEMP_BASE_DIR, "binary.lob"); - - FSDataOutputStream fsDataOutputStream = fs.create(p); - CryptoOutputStream cryptoOutputStream = spy(new CryptoOutputStream(fsDataOutputStream, new JceAesCtrCryptoCodec(), 512, BYTES, BYTES)); - - return cryptoOutputStream; - } - -} -
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/io/TestNamedFifo.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/io/TestNamedFifo.java b/src/test/com/cloudera/sqoop/io/TestNamedFifo.java deleted file mode 100644 index b11bbee..0000000 --- a/src/test/com/cloudera/sqoop/io/TestNamedFifo.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.io; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; - -import org.apache.hadoop.util.StringUtils; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.Shell; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Test the named fifo utility. - */ -public class TestNamedFifo { - - public static final Log LOG = LogFactory.getLog( - TestNamedFifo.class.getName()); - - public static final Path TEMP_BASE_DIR; - - static { - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - if (!tmpDir.endsWith(File.separator)) { - tmpDir = tmpDir + File.separator; - } - - TEMP_BASE_DIR = new Path(new Path(tmpDir), "namedfifo"); - } - - private Configuration conf; - private FileSystem fs; - - @Before - public void setUp() throws Exception { - conf = new Configuration(); - conf.set("fs.default.name", "file:///"); - - fs = FileSystem.getLocal(conf); - fs.mkdirs(TEMP_BASE_DIR); - } - - static final String MSG = "THIS IS THE MESSAGE\n"; - static final String MSG2 = "Here is a follow-up.\n"; - - private static class ReaderThread extends Thread { - private File file; - private IOException exception; - - public ReaderThread(File f) { - this.file = f; - } - - /** return any exception during the run method. */ - public IOException getException() { - return this.exception; - } - - public void run() { - BufferedReader r = null; - try { - r = new BufferedReader(new InputStreamReader( - new FileInputStream(file))); - - // Assert that after a flush, we get back what we wrote. - String line = r.readLine(); - if (!MSG.trim().equals(line)) { - throw new IOException("Expected " + MSG.trim() + " but got " - + line); - } - - // Assert that after closing the writer, we get back what - // we wrote again. - line = r.readLine(); - if (null == line) { - throw new IOException("line2 was null"); - } else if (!MSG2.trim().equals(line)) { - throw new IOException("Expected " + MSG2.trim() + " but got " - + line); - } - } catch (IOException ioe) { - this.exception = ioe; - } finally { - if (null != r) { - try { - r.close(); - } catch (IOException ioe) { - LOG.warn("Error closing reader: " + ioe); - } - } - } - } - } - - private static class WriterThread extends Thread { - private File file; - private IOException exception; - - public WriterThread(File f) { - this.file = f; - } - - /** return any exception during the run method. */ - public IOException getException() { - return this.exception; - } - - public void run() { - BufferedWriter w = null; - try { - w = new BufferedWriter(new OutputStreamWriter( - new FileOutputStream(file))); - - w.write(MSG); - w.flush(); - - w.write(MSG2); - } catch (IOException ioe) { - this.exception = ioe; - } finally { - if (null != w) { - try { - w.close(); - } catch (IOException ioe) { - LOG.warn("Error closing writer: " + ioe); - } - } - } - } - } - - @Test - public void testNamedFifo() throws Exception { - - if (Shell.WINDOWS) { - // NamedFifo uses Linux specific commands like mknod - // and mkfifo, so skip the test on Windows OS - LOG.warn("Named FIFO is not supported on Windows. Skipping test"); - return; - } - - File root = new File(TEMP_BASE_DIR.toString()); - File fifo = new File(root, "foo-fifo"); - - NamedFifo nf = new NamedFifo(fifo); - nf.create(); - - File returned = nf.getFile(); - - // These should be the same object. - assertEquals(fifo, returned); - - ReaderThread rt = new ReaderThread(returned); - WriterThread wt = new WriterThread(returned); - - rt.start(); - wt.start(); - - rt.join(); - wt.join(); - - IOException rex = rt.getException(); - IOException wex = wt.getException(); - - if (null != rex) { - LOG.error("reader exception: " + StringUtils.stringifyException(rex)); - } - - if (null != wex) { - LOG.error("writer exception: " + StringUtils.stringifyException(wex)); - } - - assertNull(rex); - assertNull(wex); - } -} - http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java b/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java deleted file mode 100644 index 8b2b1e5..0000000 --- a/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.io; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.File; -import java.io.FileInputStream; -import java.util.zip.GZIPInputStream; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.compress.GzipCodec; - -import com.cloudera.sqoop.testutil.ImportJobTestCase; - -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test that the splittable buffered writer system works. - */ -public class TestSplittableBufferedWriter { - - public static final Log LOG = LogFactory.getLog( - TestSplittableBufferedWriter.class.getName()); - - private String getWriteDir() { - return new File(ImportJobTestCase.TEMP_BASE_DIR, - "bufferedWriterTest").toString(); - } - - private Path getWritePath() { - return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest"); - } - - /** Create the directory where we'll write our test files to; and - * make sure it has no files in it. - */ - private void ensureEmptyWriteDir() throws IOException { - FileSystem fs = FileSystem.getLocal(getConf()); - Path writeDir = getWritePath(); - - fs.mkdirs(writeDir); - - FileStatus [] stats = fs.listStatus(writeDir); - - for (FileStatus stat : stats) { - if (stat.isDir()) { - fail("setUp(): Write directory " + writeDir - + " contains subdirectories"); - } - - LOG.debug("setUp(): Removing " + stat.getPath()); - if (!fs.delete(stat.getPath(), false)) { - fail("setUp(): Could not delete residual file " + stat.getPath()); - } - } - - if (!fs.exists(writeDir)) { - fail("setUp: Could not create " + writeDir); - } - } - - @Before - public void setUp() throws IOException { - ensureEmptyWriteDir(); - } - - private Configuration getConf() { - Configuration conf = new Configuration(); - conf.set("fs.default.name", "file:///"); - return conf; - } - - /** Verifies contents of an InputStream. Closes the InputStream on - * its way out. Fails the test if the file doesn't match the expected set - * of lines. - */ - private void verifyFileContents(InputStream is, String [] lines) - throws IOException { - BufferedReader r = new BufferedReader(new InputStreamReader(is)); - try { - for (String expectedLine : lines) { - String actualLine = r.readLine(); - assertNotNull(actualLine); - assertEquals("Input line mismatch", expectedLine, actualLine); - } - - assertNull("Stream had additional contents after expected line", - r.readLine()); - } finally { - r.close(); - try { - is.close(); - } catch (IOException ioe) { - // ignore IOE; may be closed by reader. - } - } - } - - private void verifyFileExists(Path p) throws IOException { - FileSystem fs = FileSystem.getLocal(getConf()); - assertTrue("File not found: " + p, fs.exists(p)); - } - - private void verifyFileDoesNotExist(Path p) throws IOException { - FileSystem fs = FileSystem.getLocal(getConf()); - assertFalse("File found: " + p + " and we did not expect it", fs.exists(p)); - } - - @Test - public void testNonSplittingTextFile() throws IOException { - SplittingOutputStream os = new SplittingOutputStream(getConf(), - getWritePath(), "nonsplit-", 0, null); - try { - SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); - try { - w.allowSplit(); - w.write("This is a string!"); - w.newLine(); - w.write("This is another string!"); - w.allowSplit(); - } finally { - w.close(); - } - } finally { - try { - os.close(); - } catch (IOException ioe) { - // Ignored; may be thrown because w is already closed. - } - } - - // Ensure we made exactly one file. - Path writePath = new Path(getWritePath(), "nonsplit-00000"); - Path badPath = new Path(getWritePath(), "nonsplit-00001"); - verifyFileExists(writePath); - verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file. - - // Now ensure all the data got there. - String [] expectedLines = { - "This is a string!", - "This is another string!", - }; - - InputStream fis = new FileInputStream(new File(getWriteDir(), - "nonsplit-00000")); - try { - verifyFileContents(fis, expectedLines); - } finally { - try { - fis.close(); - } catch (IOException ioe) { - // Ignored; may be closed by verifyFileContents(). - } - } - } - - @Test - public void testNonSplittingGzipFile() throws IOException { - SplittingOutputStream os = new SplittingOutputStream(getConf(), - getWritePath(), "nonsplit-", 0, new GzipCodec()); - SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); - try { - w.allowSplit(); - w.write("This is a string!"); - w.newLine(); - w.write("This is another string!"); - w.allowSplit(); - } finally { - w.close(); - } - - // Ensure we made exactly one file. - Path writePath = new Path(getWritePath(), "nonsplit-00000.gz"); - Path badPath = new Path(getWritePath(), "nonsplit-00001.gz"); - verifyFileExists(writePath); - verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file. - - // Now ensure all the data got there. - String [] expectedLines = { - "This is a string!", - "This is another string!", - }; - verifyFileContents( - new GZIPInputStream(new FileInputStream(new File(getWriteDir(), - "nonsplit-00000.gz"))), expectedLines); - } - - @Test - public void testSplittingTextFile() throws IOException { - SplittingOutputStream os = new SplittingOutputStream(getConf(), - getWritePath(), "split-", 10, null); - try { - SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); - try { - w.allowSplit(); - w.write("This is a string!"); - w.newLine(); - w.write("This is another string!"); - } finally { - w.close(); - } - } finally { - try { - os.close(); - } catch (IOException ioe) { - // Ignored; may be thrown because w is already closed. - } - } - - // Ensure we made exactly two files. - Path writePath = new Path(getWritePath(), "split-00000"); - Path writePath2 = new Path(getWritePath(), "split-00001"); - Path badPath = new Path(getWritePath(), "split-00002"); - verifyFileExists(writePath); - verifyFileExists(writePath2); - verifyFileDoesNotExist(badPath); // Ensure we didn't make three files. - - // Now ensure all the data got there. - String [] expectedLines0 = { - "This is a string!", - }; - InputStream fis = new FileInputStream(new File(getWriteDir(), - "split-00000")); - try { - verifyFileContents(fis, expectedLines0); - } finally { - try { - fis.close(); - } catch (IOException ioe) { - // ignored; may be generated because fis closed in verifyFileContents. - } - } - - String [] expectedLines1 = { - "This is another string!", - }; - fis = new FileInputStream(new File(getWriteDir(), "split-00001")); - try { - verifyFileContents(fis, expectedLines1); - } finally { - try { - fis.close(); - } catch (IOException ioe) { - // Ignored; may be thrown because it's closed in verifyFileContents. - } - } - } - - @Test - public void testSplittingGzipFile() throws IOException { - SplittingOutputStream os = new SplittingOutputStream(getConf(), - getWritePath(), "splitz-", 3, new GzipCodec()); - SplittableBufferedWriter w = new SplittableBufferedWriter(os, true); - try { - w.write("This is a string!"); - w.newLine(); - w.write("This is another string!"); - } finally { - w.close(); - } - - // Ensure we made exactly two files. - Path writePath = new Path(getWritePath(), "splitz-00000.gz"); - Path writePath2 = new Path(getWritePath(), "splitz-00001.gz"); - Path badPath = new Path(getWritePath(), "splitz-00002.gz"); - verifyFileExists(writePath); - verifyFileExists(writePath2); - verifyFileDoesNotExist(badPath); // Ensure we didn't make three files. - - // Now ensure all the data got there. - String [] expectedLines0 = { - "This is a string!", - }; - verifyFileContents( - new GZIPInputStream(new FileInputStream(new File(getWriteDir(), - "splitz-00000.gz"))), expectedLines0); - - String [] expectedLines1 = { - "This is another string!", - }; - verifyFileContents( - new GZIPInputStream(new FileInputStream(new File(getWriteDir(), - "splitz-00001.gz"))), expectedLines1); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestBlobRef.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestBlobRef.java b/src/test/com/cloudera/sqoop/lib/TestBlobRef.java deleted file mode 100644 index 0d010b0..0000000 --- a/src/test/com/cloudera/sqoop/lib/TestBlobRef.java +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.lib; - -import java.io.*; - -import com.cloudera.sqoop.testutil.BaseSqoopTestCase; -import com.cloudera.sqoop.testutil.CommonArgs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import com.cloudera.sqoop.io.LobFile; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * Test that the BlobRef.parse() method does the right thing. - * Note that we don't support inline parsing here; we only expect this to - * really work for external BLOBs. - */ -public class TestBlobRef { - - @Test - public void testEmptyStr() { - BlobRef r = BlobRef.parse(""); - assertFalse(r.isExternal()); - } - - @Test - public void testInline() throws IOException { - BlobRef r = BlobRef.parse("foo"); - assertFalse(r.isExternal()); - } - - @Test - public void testEmptyFile() { - BlobRef r = BlobRef.parse("externalLob()"); - assertFalse(r.isExternal()); - - r = BlobRef.parse("externalLob(lf,,0,0)"); - assertTrue(r.isExternal()); - assertEquals("externalLob(lf,,0,0)", r.toString()); - } - - @Test - public void testInlineNearMatch() { - BlobRef r = BlobRef.parse("externalLob(foo)bar"); - assertFalse(r.isExternal()); - - r = BlobRef.parse("externalLob(foo)"); - assertFalse(r.isExternal()); - - r = BlobRef.parse("externalLob(lf,foo)"); - assertFalse(r.isExternal()); - - r = BlobRef.parse("externalLob(lf,foo,1,2)x"); - assertFalse(r.isExternal()); - } - - @Test - public void testExternal() throws IOException { - final byte [] DATA = { 1, 2, 3, 4, 5 }; - final String FILENAME = "blobdata"; - - doExternalTest(DATA, FILENAME); - } - - @Test - public void testExternalSubdir() throws IOException { - final byte [] DATA = { 1, 2, 3, 4, 5 }; - final String FILENAME = "_lob/blobdata"; - - try { - doExternalTest(DATA, FILENAME); - } finally { - // remove dir we made. - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.getLocal(conf); - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - Path lobDir = new Path(new Path(tmpDir), "_lob"); - fs.delete(lobDir, true); - } - } - - private void doExternalTest(final byte [] data, final String filename) - throws IOException { - - Configuration conf = new Configuration(); - if (!BaseSqoopTestCase.isOnPhysicalCluster()) { - conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); - } - FileSystem fs = FileSystem.get(conf); - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - - Path tmpPath = new Path(tmpDir); - Path blobFile = new Path(tmpPath, filename); - - // make any necessary parent dirs. - Path blobParent = blobFile.getParent(); - if (!fs.exists(blobParent)) { - fs.mkdirs(blobParent); - } - - LobFile.Writer lw = LobFile.create(blobFile, conf, false); - try { - long off = lw.tell(); - long len = data.length; - OutputStream os = lw.writeBlobRecord(len); - os.write(data, 0, data.length); - os.close(); - lw.close(); - - String refString = "externalLob(lf," + filename - + "," + off + "," + len + ")"; - BlobRef blob = BlobRef.parse(refString); - assertTrue(blob.isExternal()); - assertEquals(refString, blob.toString()); - InputStream is = blob.getDataStream(conf, tmpPath); - assertNotNull(is); - - byte [] buf = new byte[4096]; - int bytes = is.read(buf, 0, 4096); - is.close(); - - assertEquals(data.length, bytes); - for (int i = 0; i < bytes; i++) { - assertEquals(data[i], buf[i]); - } - } finally { - fs.delete(blobFile, false); - } - } -} - http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java b/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java deleted file mode 100644 index c1ab7e5..0000000 --- a/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.lib; - - -import org.junit.Test; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test the boolean parser. - */ -public class TestBooleanParser { - - @Test - public void testBoolParser() { - assertTrue(BooleanParser.valueOf("true")); - assertTrue(BooleanParser.valueOf("TRUE")); - assertTrue(BooleanParser.valueOf("True")); - assertTrue(BooleanParser.valueOf("t")); - assertTrue(BooleanParser.valueOf("T")); - assertTrue(BooleanParser.valueOf("on")); - assertTrue(BooleanParser.valueOf("On")); - assertTrue(BooleanParser.valueOf("ON")); - assertTrue(BooleanParser.valueOf("yes")); - assertTrue(BooleanParser.valueOf("yEs")); - assertTrue(BooleanParser.valueOf("YES")); - assertTrue(BooleanParser.valueOf("1")); - - assertFalse(BooleanParser.valueOf(null)); - - assertFalse(BooleanParser.valueOf("no")); - assertFalse(BooleanParser.valueOf("false")); - assertFalse(BooleanParser.valueOf("FALSE")); - assertFalse(BooleanParser.valueOf("0")); - assertFalse(BooleanParser.valueOf("off")); - assertFalse(BooleanParser.valueOf("OFF")); - assertFalse(BooleanParser.valueOf("anything else in the world")); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestClobRef.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestClobRef.java b/src/test/com/cloudera/sqoop/lib/TestClobRef.java deleted file mode 100644 index 7efc3b6..0000000 --- a/src/test/com/cloudera/sqoop/lib/TestClobRef.java +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.lib; - -import java.io.*; - -import com.cloudera.sqoop.testutil.BaseSqoopTestCase; -import com.cloudera.sqoop.testutil.CommonArgs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import com.cloudera.sqoop.io.LobFile; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * Test parsing of ClobRef objects. - */ -public class TestClobRef { - - @Test - public void testEmptyStr() { - ClobRef r = ClobRef.parse(""); - assertFalse(r.isExternal()); - assertEquals("", r.toString()); - } - - @Test - public void testInline() throws IOException { - ClobRef r = ClobRef.parse("foo"); - assertFalse(r.isExternal()); - assertEquals("foo", r.toString()); - - Reader reader = r.getDataStream(null, null); - assertNotNull(reader); - char [] buf = new char[4096]; - int chars = reader.read(buf, 0, 4096); - reader.close(); - - String str = new String(buf, 0, chars); - assertEquals("foo", str); - } - - @Test - public void testEmptyFile() { - ClobRef r = ClobRef.parse("externalLob()"); - assertFalse(r.isExternal()); - assertEquals("externalLob()", r.toString()); - - r = ClobRef.parse("externalLob(lf,,0,0)"); - assertTrue(r.isExternal()); - assertEquals("externalLob(lf,,0,0)", r.toString()); - } - - @Test - public void testInlineNearMatch() { - ClobRef r = ClobRef.parse("externalLob(foo)bar"); - assertFalse(r.isExternal()); - assertEquals("externalLob(foo)bar", r.toString()); - - r = ClobRef.parse("externalLob(foo)"); - assertFalse(r.isExternal()); - assertEquals("externalLob(foo)", r.getData()); - - r = ClobRef.parse("externalLob(lf,foo)"); - assertFalse(r.isExternal()); - assertEquals("externalLob(lf,foo)", r.getData()); - - r = ClobRef.parse("externalLob(lf,foo,1,2)x"); - assertFalse(r.isExternal()); - assertEquals("externalLob(lf,foo,1,2)x", r.getData()); - } - - @Test - public void testExternal() throws IOException { - final String DATA = "This is the clob data!"; - final String FILENAME = "clobdata"; - - doExternalTest(DATA, FILENAME); - } - - @Test - public void testExternalSubdir() throws IOException { - final String DATA = "This is the clob data!"; - final String FILENAME = "_lob/clobdata"; - - try { - doExternalTest(DATA, FILENAME); - } finally { - // remove dir we made. - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.getLocal(conf); - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - Path lobDir = new Path(new Path(tmpDir), "_lob"); - fs.delete(lobDir, true); - } - } - - private void doExternalTest(final String data, final String filename) - throws IOException { - - Configuration conf = new Configuration(); - if (!BaseSqoopTestCase.isOnPhysicalCluster()) { - conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); - } - FileSystem fs = FileSystem.get(conf); - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - - Path tmpPath = new Path(tmpDir); - Path clobFile = new Path(tmpPath, filename); - - // make any necessary parent dirs. - Path clobParent = clobFile.getParent(); - if (!fs.exists(clobParent)) { - fs.mkdirs(clobParent); - } - - LobFile.Writer lw = LobFile.create(clobFile, conf, true); - try { - long off = lw.tell(); - long len = data.length(); - Writer w = lw.writeClobRecord(len); - w.append(data); - w.close(); - lw.close(); - - String refString = "externalLob(lf," + filename - + "," + off + "," + len + ")"; - ClobRef clob = ClobRef.parse(refString); - assertTrue(clob.isExternal()); - assertEquals(refString, clob.toString()); - Reader r = clob.getDataStream(conf, tmpPath); - assertNotNull(r); - - char [] buf = new char[4096]; - int chars = r.read(buf, 0, 4096); - r.close(); - - String str = new String(buf, 0, chars); - assertEquals(data, str); - } finally { - fs.delete(clobFile, false); - } - } -} - http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java b/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java deleted file mode 100644 index 1fc9cd2..0000000 --- a/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java +++ /dev/null @@ -1,150 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.lib; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - - -/** - * Test that the field formatter works in a variety of configurations. - */ -public class TestFieldFormatter { - - @Test - public void testAllEmpty() { - String result = FieldFormatter.escapeAndEnclose("", - new DelimiterSet(DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR, - DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR, false)); - assertEquals("", result); - } - - @Test - public void testNullArgs() { - assertNull(FieldFormatter.escapeAndEnclose(null, - new DelimiterSet('\"', DelimiterSet.NULL_CHAR, '\"', '\\', false))); - } - - @Test - public void testBasicStr() { - String result = FieldFormatter.escapeAndEnclose("foo", - DelimiterSet.DEFAULT_DELIMITERS); - assertEquals("foo", result); - } - - @Test - public void testEscapeSlash() { - String result = FieldFormatter.escapeAndEnclose("foo\\bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("foo\\\\bar", result); - } - - @Test - public void testMustEnclose() { - String result = FieldFormatter.escapeAndEnclose("foo", - new DelimiterSet(',', '\n', '\"', DelimiterSet.NULL_CHAR, true)); - assertEquals("\"foo\"", result); - } - - @Test - public void testEncloseComma1() { - String result = FieldFormatter.escapeAndEnclose("foo,bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("\"foo,bar\"", result); - } - - @Test - public void testEncloseComma2() { - String result = FieldFormatter.escapeAndEnclose("foo,bar", - new DelimiterSet(',', ',', '\"', '\\', false)); - assertEquals("\"foo,bar\"", result); - } - - @Test - public void testNoNeedToEnclose() { - String result = FieldFormatter.escapeAndEnclose( - "just another string", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("just another string", result); - } - - @Test - public void testCannotEnclose() { - // Can't enclose because encloser is nul. - // This should escape the comma instead. - String result = FieldFormatter.escapeAndEnclose("foo,bar", - new DelimiterSet(',', '\n', DelimiterSet.NULL_CHAR, '\\', false)); - - assertEquals("foo\\,bar", result); - } - - @Test - public void testEmptyCharToEscapeString() { - // test what happens when the escape char is null. It should encode the - // null char. - - char nul = DelimiterSet.NULL_CHAR; - String s = "" + nul; - assertEquals("\000", s); - } - - @Test - public void testEscapeCentralQuote() { - String result = FieldFormatter.escapeAndEnclose("foo\"bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("foo\\\"bar", result); - } - - @Test - public void testEscapeMultiCentralQuote() { - String result = FieldFormatter.escapeAndEnclose("foo\"\"bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("foo\\\"\\\"bar", result); - } - - @Test - public void testDoubleEscape() { - String result = FieldFormatter.escapeAndEnclose("foo\\\"bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("foo\\\\\\\"bar", result); - } - - @Test - public void testReverseEscape() { - String result = FieldFormatter.escapeAndEnclose("foo\"\\bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("foo\\\"\\\\bar", result); - } - - @Test - public void testQuotedEncloser() { - String result = FieldFormatter.escapeAndEnclose("foo\",bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("\"foo\\\",bar\"", result); - } - - @Test - public void testQuotedEscape() { - String result = FieldFormatter.escapeAndEnclose("foo\\,bar", - new DelimiterSet(',', '\n', '\"', '\\', false)); - assertEquals("\"foo\\\\,bar\"", result); - } -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java b/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java deleted file mode 100644 index c55f14b..0000000 --- a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.lib; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.sql.ResultSet; -import java.sql.SQLException; - -import com.cloudera.sqoop.testutil.BaseSqoopTestCase; -import com.cloudera.sqoop.testutil.CommonArgs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import com.cloudera.sqoop.testutil.MockResultSet; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * Test deserialization of ClobRef and BlobRef fields. - */ -public class TestLargeObjectLoader { - - protected Configuration conf; - protected LargeObjectLoader loader; - protected Path outDir; - - @Before - public void setUp() throws IOException, InterruptedException { - conf = new Configuration(); - if (!BaseSqoopTestCase.isOnPhysicalCluster()) { - conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); - } - String tmpDir = System.getProperty("test.build.data", "/tmp/"); - this.outDir = new Path(System.getProperty("java.io.tmpdir")); - FileSystem fs = FileSystem.get(conf); - if (fs.exists(outDir)) { - fs.delete(outDir, true); - } - fs.mkdirs(outDir); - - loader = new LargeObjectLoader(conf, outDir); - } - - @Test - public void testReadClobRef() - throws IOException, InterruptedException, SQLException { - // This should give us an inline CLOB. - ResultSet resultSet = new MockResultSet(); - ClobRef clob = loader.readClobRef(0, resultSet); - assertNotNull(clob); - assertFalse(clob.isExternal()); - assertEquals(MockResultSet.CLOB_DATA, clob.toString()); - - // LOBs bigger than 4 bytes are now external. - conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4); - clob = loader.readClobRef(0, resultSet); - assertNotNull(clob); - assertTrue(clob.isExternal()); - loader.close(); - Reader r = clob.getDataStream(conf, outDir); - char [] buf = new char[4096]; - int chars = r.read(buf, 0, 4096); - r.close(); - String str = new String(buf, 0, chars); - assertEquals(MockResultSet.CLOB_DATA, str); - } - - @Test - public void testReadBlobRef() - throws IOException, InterruptedException, SQLException { - // This should give us an inline BLOB. - ResultSet resultSet = new MockResultSet(); - BlobRef blob = loader.readBlobRef(0, resultSet); - assertNotNull(blob); - assertFalse(blob.isExternal()); - byte [] data = blob.getData(); - byte [] blobData = MockResultSet.blobData(); - assertEquals(blobData.length, data.length); - for (int i = 0; i < data.length; i++) { - assertEquals(blobData[i], data[i]); - } - - // LOBs bigger than 4 bytes are now external. - conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4); - blob = loader.readBlobRef(0, resultSet); - assertNotNull(blob); - assertTrue(blob.isExternal()); - loader.close(); - InputStream is = blob.getDataStream(conf, outDir); - byte [] buf = new byte[4096]; - int bytes = is.read(buf, 0, 4096); - is.close(); - - assertEquals(blobData.length, bytes); - for (int i = 0; i < bytes; i++) { - assertEquals(blobData[i], buf[i]); - } - } -} - http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestRecordParser.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestRecordParser.java b/src/test/com/cloudera/sqoop/lib/TestRecordParser.java deleted file mode 100644 index d964cef..0000000 --- a/src/test/com/cloudera/sqoop/lib/TestRecordParser.java +++ /dev/null @@ -1,450 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.lib; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.fail; - - -/** - * Test that the record parser works in a variety of configurations. - */ -public class TestRecordParser { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - private void assertListsEqual(String msg, List<String> expected, - List<String> actual) { - if (expected == null && actual != null) { - if (null == msg) { - msg = "expected null list"; - } - - fail(msg); - } else if (expected != null && actual == null) { - if (null == msg) { - msg = "expected non-null list"; - } - - fail(msg); - } else if (expected == null && actual == null) { - return; // ok. Both null; nothing to do. - } - - assert(null != expected); - assert(null != actual); - - int expectedLen = expected.size(); - int actualLen = actual.size(); - - if (expectedLen != actualLen) { - if (null == msg) { - msg = "Expected list of length " + expectedLen - + "; got " + actualLen; - } - - fail(msg); - } - - // Check the list contents. - for (int i = 0; i < expectedLen; i++) { - String expectedElem = expected.get(i); - String actualElem = actual.get(i); - - if (expectedElem == null) { - if (actualElem != null) { - if (null == msg) { - msg = "Expected null element at position " + i - + "; got [" + actualElem + "]"; - } - - fail(msg); - } - } else if (!expectedElem.equals(actualElem)) { - if (null == msg) { - msg = "Expected [" + expectedElem + "] at position " + i - + "; got [" + actualElem + "]"; - } - - fail(msg); - } - } - } - - private List<String> list(String [] items) { - - if (null == items) { - return null; - } - - ArrayList<String> asList = new ArrayList<String>(); - for (int i = 0; i < items.length; i++) { - asList.add(items[i]); - } - - return asList; - } - - @Test - public void testEmptyLine() throws RecordParser.ParseError { - // an empty line should return no fields. - - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { }; - assertListsEqual(null, list(strings), parser.parseRecord("")); - } - - @Test - public void testJustEOR() throws RecordParser.ParseError { - // a line with just a newline char should return a single zero-length field. - - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "" }; - assertListsEqual(null, list(strings), parser.parseRecord("\n")); - } - - @Test - public void testOneField() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the field" }; - assertListsEqual(null, list(strings), parser.parseRecord("the field")); - } - - @Test - public void testOneField2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the field" }; - assertListsEqual(null, list(strings), parser.parseRecord("the field\n")); - } - - @Test - public void testQuotedField1() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the field" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the field\"\n")); - } - - @Test - public void testQuotedField2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the field" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the field\"")); - } - - @Test - public void testQuotedField3() throws RecordParser.ParseError { - // quoted containing EOF - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the ,field" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the ,field\"")); - } - - @Test - public void testQuotedField4() throws RecordParser.ParseError { - // quoted containing multiple EOFs - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the ,,field" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the ,,field\"")); - } - - @Test - public void testQuotedField5() throws RecordParser.ParseError { - // quoted containing EOF and EOR - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the ,\nfield" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the ,\nfield\"")); - } - - @Test - public void testQuotedField6() throws RecordParser.ParseError { - // quoted containing EOR - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the \nfield" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the \nfield\"")); - } - - @Test - public void testQuotedField7() throws RecordParser.ParseError { - // quoted containing multiple EORs - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the \n\nfield" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the \n\nfield\"")); - } - - @Test - public void testQuotedField8() throws RecordParser.ParseError { - // quoted containing escaped quoted char - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the \"field" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"the \\\"field\"")); - } - - @Test - public void testUnquotedEscape1() throws RecordParser.ParseError { - // field without quotes with an escaped EOF char. - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the ,field" }; - assertListsEqual(null, list(strings), parser.parseRecord("the \\,field")); - } - - @Test - public void testUnquotedEscape2() throws RecordParser.ParseError { - // field without quotes with an escaped escape char. - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "the \\field" }; - assertListsEqual(null, list(strings), parser.parseRecord("the \\\\field")); - } - - @Test - public void testTwoFields1() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), parser.parseRecord("field1,field2")); - } - - @Test - public void testTwoFields2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), - parser.parseRecord("field1,field2\n")); - } - - @Test - public void testTwoFields3() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"field1\",field2\n")); - } - - @Test - public void testTwoFields4() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), - parser.parseRecord("field1,\"field2\"\n")); - } - - @Test - public void testTwoFields5() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), - parser.parseRecord("field1,\"field2\"")); - } - - @Test - public void testRequiredQuotes0() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', true)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"field1\",\"field2\"\n")); - } - - @Test - public void testRequiredQuotes1() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', true)); - String [] strings = { "field1", "field2" }; - assertListsEqual(null, list(strings), - parser.parseRecord("\"field1\",\"field2\"")); - } - - @Test - public void testRequiredQuotes2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', true)); - - thrown.expect(RecordParser.ParseError.class); - thrown.reportMissingExceptionWithMessage("Expected parse error for required quotes"); - parser.parseRecord("\"field1\",field2"); - } - - @Test - public void testRequiredQuotes3() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', true)); - - thrown.expect(RecordParser.ParseError.class); - thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes"); - parser.parseRecord("field1,\"field2\""); - } - - @Test - public void testRequiredQuotes4() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', true)); - - thrown.expect(RecordParser.ParseError.class); - thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes"); - parser.parseRecord("field1,\"field2\"\n"); - } - - @Test - public void testNull() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', true)); - String input = null; - - thrown.expect(RecordParser.ParseError.class); - thrown.reportMissingExceptionWithMessage("Expected ParseError for null string"); - parser.parseRecord(input); - } - - - @Test - public void testEmptyFields1() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "", ""}; - assertListsEqual(null, list(strings), parser.parseRecord(",")); - } - - @Test - public void testEmptyFields2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "", "" }; - assertListsEqual(null, list(strings), parser.parseRecord(",\n")); - } - - @Test - public void testEmptyFields3() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "", "", "" }; - assertListsEqual(null, list(strings), parser.parseRecord(",,\n")); - } - - @Test - public void testEmptyFields4() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "", "foo", "" }; - assertListsEqual(null, list(strings), parser.parseRecord(",foo,\n")); - } - - @Test - public void testEmptyFields5() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "", "foo", "" }; - assertListsEqual(null, list(strings), parser.parseRecord(",foo,")); - } - - @Test - public void testEmptyFields6() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "foo", "" }; - assertListsEqual(null, list(strings), parser.parseRecord("foo,")); - } - - @Test - public void testTrailingText() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "foo", "bar" }; - assertListsEqual(null, list(strings), parser.parseRecord("foo,bar\nbaz")); - } - - @Test - public void testTrailingText2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "" }; - assertListsEqual(null, list(strings), parser.parseRecord("\nbaz")); - } - - @Test - public void testLeadingEscape() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', '\n', '\"', '\\', false)); - String [] strings = { "\nbaz" }; - assertListsEqual(null, list(strings), parser.parseRecord("\\\nbaz")); - } - - @Test - public void testEofIsEor() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', ',', '\"', '\\', false)); - String [] strings = { "three", "different", "fields" }; - assertListsEqual(null, list(strings), - parser.parseRecord("three,different,fields")); - } - - @Test - public void testEofIsEor2() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', ',', '\"', '\\', false)); - String [] strings = { "three", "different", "fields" }; - assertListsEqual(null, list(strings), - parser.parseRecord("three,\"different\",fields")); - } - - @Test - public void testRepeatedParse() throws RecordParser.ParseError { - RecordParser parser = new RecordParser( - new DelimiterSet(',', ',', '\"', '\\', false)); - String [] strings = { "three", "different", "fields" }; - assertListsEqual(null, list(strings), - parser.parseRecord("three,\"different\",fields")); - - String [] strings2 = { "foo", "bar" }; - assertListsEqual(null, list(strings2), - parser.parseRecord("foo,\"bar\"")); - } - -} http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java b/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java deleted file mode 100644 index 36aa821..0000000 --- a/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java +++ /dev/null @@ -1,306 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.cloudera.sqoop.manager; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.Writer; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.util.StringUtils; -import org.apache.sqoop.manager.CubridManager; -import org.apache.sqoop.manager.cubrid.CubridTestUtils; -import org.junit.After; -import org.junit.Before; - -import com.cloudera.sqoop.SqoopOptions; -import com.cloudera.sqoop.TestExport; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * Test the CubridManager implementation. - * - * This uses JDBC to export data from HDFS to an Cubrid database. - * - * Since this requires an Cubrid installation, this class is named in such a way - * that Sqoop's default QA process does not run it. You need to run this - * manually with -Dtestcase=CubridManagerExportTest. - * - * You need to put Cubrid JDBC driver library (JDBC-9.2.2.0003-cubrid.jar) in a - * location where Sqoop will be able to access it (since this library cannot be - * checked into Apache's tree for licensing reasons). - * - * To set up your test environment: - * Install Cubrid 9.2.2 - * ref:http://www.cubrid.org/wiki_tutorials/entry/installing-cubrid-on-linux-using-shell-and-rpm - * Create a database SQOOPCUBRIDTEST - * $cubrid createdb SQOOPCUBRIDTEST en_us.utf8 - * Start cubrid and database - * $cubrid service start - * $cubrid server start SQOOPCUBRIDTEST - * Create a login SQOOPUSER with password PASSWORD and grant all - * $csql -u dba SQOOPCUBRIDTEST - * csql>CREATE USER SQOOPUSER password 'PASSWORD'; - */ -public class CubridManagerExportTest extends TestExport { - - public static final Log LOG = LogFactory.getLog( - CubridManagerExportTest.class.getName()); - - static final String TABLE_PREFIX = "EXPORT_CUBRID_"; - - // instance variables populated during setUp, used during tests. - private CubridManager manager; - private Connection conn; - - @Override - protected Connection getConnection() { - return conn; - } - - @Override - protected boolean useHsqldbTestServer() { - return false; - } - - @Override - protected String getConnectString() { - return CubridTestUtils.getConnectString(); - } - - @Override - protected String getTablePrefix() { - return TABLE_PREFIX; - } - - @Override - protected String getDropTableStatement(String tableName) { - return "DROP TABLE IF EXISTS " + tableName; - } - - /** - * Cubrid could not support --staging-table, Diable this test case. - */ - @Override - @Test - public void testMultiTransactionWithStaging() throws IOException, - SQLException { - return; - } - - /** - * Cubrid could not support --staging-table, Diable this test case. - */ - @Override - @Test - public void testMultiMapTextExportWithStaging() throws IOException, - SQLException { - return; - } - - public void createTableAndPopulateData(String table) { - String fulltableName = manager.escapeTableName(table); - - Statement stmt = null; - - // Drop the existing table, if there is one. - try { - conn = manager.getConnection(); - stmt = conn.createStatement(); - stmt.execute("DROP TABLE IF EXISTS " + fulltableName); - conn.commit(); - } catch (SQLException sqlE) { - LOG.info("Table was not dropped: " + sqlE.getMessage()); - } finally { - try { - if (null != stmt) { - stmt.close(); - } - } catch (Exception ex) { - LOG.warn("Exception while closing stmt", ex); - } - } - - // Create and populate table - try { - conn = manager.getConnection(); - conn.setAutoCommit(false); - stmt = conn.createStatement(); - - // create the database table and populate it with data. - stmt.executeUpdate("CREATE TABLE " - + fulltableName + " (" - + "id INT NOT NULL, " - + "name VARCHAR(24) NOT NULL, " - + "salary FLOAT, " + "dept VARCHAR(32), " - + "PRIMARY KEY (id))"); - conn.commit(); - } catch (SQLException sqlE) { - LOG.error("Encountered SQL Exception: ", sqlE); - sqlE.printStackTrace(); - fail("SQLException when running test setUp(): " + sqlE); - } finally { - try { - if (null != stmt) { - stmt.close(); - } - } catch (Exception ex) { - LOG.warn( - "Exception while closing connection/stmt", ex); - } - } - } - - @Before - public void setUp() { - super.setUp(); - - SqoopOptions options = new SqoopOptions( - CubridTestUtils.getConnectString(), - getTableName()); - options.setUsername(CubridTestUtils.getCurrentUser()); - options.setPassword(CubridTestUtils.getPassword()); - this.manager = new CubridManager(options); - try { - this.conn = manager.getConnection(); - this.conn.setAutoCommit(false); - - } catch (SQLException sqlE) { - LOG.error(StringUtils.stringifyException(sqlE)); - fail("Failed with sql exception in setup: " + sqlE); - } - } - - @After - public void tearDown() { - super.tearDown(); - try { - conn.close(); - manager.close(); - } catch (SQLException sqlE) { - LOG.error("Got SQLException: " + sqlE.toString()); - fail("Got SQLException: " + sqlE.toString()); - } - } - - @Override - protected String[] getCodeGenArgv(String... extraArgs) { - String[] moreArgs = new String[extraArgs.length + 4]; - int i = 0; - for (i = 0; i < extraArgs.length; i++) { - moreArgs[i] = extraArgs[i]; - } - - // Add username and password args. - moreArgs[i++] = "--username"; - moreArgs[i++] = CubridTestUtils.getCurrentUser(); - moreArgs[i++] = "--password"; - moreArgs[i++] = CubridTestUtils.getPassword(); - - return super.getCodeGenArgv(moreArgs); - } - - @Override - protected String[] getArgv(boolean includeHadoopFlags, - int rowsPerStatement, - int statementsPerTx, String... additionalArgv) { - - String[] subArgv = newStrArray(additionalArgv, "--username", - CubridTestUtils.getCurrentUser(), "--password", - CubridTestUtils.getPassword()); - return super.getArgv(includeHadoopFlags, rowsPerStatement, - statementsPerTx, subArgv); - } - - protected void createTestFile(String filename, - String[] lines) - throws IOException { - File testdir = new File(getWarehouseDir()); - if (!testdir.exists()) { - testdir.mkdirs(); - } - File file = new File(getWarehouseDir() + "/" + filename); - Writer output = new BufferedWriter(new FileWriter(file)); - for (String line : lines) { - output.write(line); - output.write("\n"); - } - output.close(); - } - - public static void assertRowCount(long expected, - String tableName, - Connection connection) { - Statement stmt = null; - ResultSet rs = null; - try { - stmt = connection.createStatement(); - rs = stmt.executeQuery("SELECT count(*) FROM " - + tableName); - rs.next(); - assertEquals(expected, rs.getLong(1)); - } catch (SQLException e) { - LOG.error("Can't verify number of rows", e); - fail(); - } finally { - try { - connection.commit(); - if (stmt != null) { - stmt.close(); - } - if (rs != null) { - rs.close(); - } - } catch (SQLException ex) { - LOG.info("Ignored exception in finally block."); - } - } - } - - public String escapeTableOrSchemaName(String tableName) { - return "\"" + tableName + "\""; - } - - /** Make sure mixed update/insert export work correctly. */ - @Test - public void testUpsertTextExport() throws IOException, SQLException { - final int TOTAL_RECORDS = 10; - createTextFile(0, TOTAL_RECORDS, false); - createTable(); - // first time will be insert. - runExport(getArgv(true, 10, 10, - newStrArray(null, "--update-key", "id", - "--update-mode", "allowinsert"))); - // second time will be update. - runExport(getArgv(true, 10, 10, - newStrArray(null, "--update-key", "id", - "--update-mode", "allowinsert"))); - verifyExport(TOTAL_RECORDS); - } -}
