http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java new file mode 100644 index 0000000..fc51e31 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java @@ -0,0 +1,901 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.internal.AssumptionViolatedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Properties; +import java.util.UUID; + +/** + * Utilities used across test cases to make assertions about filesystems + * -assertions which fail with useful information. + * This is lifted from Hadoop common Test; that JAR isn't published, so + * we have to make do. + */ +public class ContractTestUtils extends Assert { + + private static final Logger LOG = + LoggerFactory.getLogger(ContractTestUtils.class); + + public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; + + // For scale testing, we can repeatedly write small chunk data to generate + // a large file. + public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size"; + public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128; + public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size"; + public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128; + + /** + * Assert that a property in the property set matches the expected value + * @param props property set + * @param key property name + * @param expected expected value. If null, the property must not be in the set + */ + public static void assertPropertyEquals(Properties props, + String key, + String expected) { + String val = props.getProperty(key); + if (expected == null) { + assertNull("Non null property " + key + " = " + val, val); + } else { + assertEquals("property " + key + " = " + val, + expected, + val); + } + } + + /** + * + * Write a file and read it in, validating the result. Optional flags control + * whether file overwrite operations should be enabled, and whether the + * file should be deleted afterwards. + * + * If there is a mismatch between what was written and what was expected, + * a small range of bytes either side of the first error are logged to aid + * diagnosing what problem occurred -whether it was a previous file + * or a corrupting of the current file. This assumes that two + * sequential runs to the same path use datasets with different character + * moduli. + * + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @param delete should the file be deleted afterwards? -with a verification + * that it worked. Deletion is not attempted if an assertion has failed + * earlier -it is not in a <code>finally{}</code> block. + * @throws IOException IO problems + */ + public static void writeAndRead(FileSystem fs, + Path path, + byte[] src, + int len, + int blocksize, + boolean overwrite, + boolean delete) throws IOException { + fs.mkdirs(path.getParent()); + + writeDataset(fs, path, src, len, blocksize, overwrite); + + byte[] dest = readDataset(fs, path, len); + + compareByteArrays(src, dest, len); + + if (delete) { + rejectRootOperation(path); + boolean deleted = fs.delete(path, false); + assertTrue("Deleted", deleted); + assertPathDoesNotExist(fs, "Cleanup failed", path); + } + } + + /** + * Write a file. + * Optional flags control + * whether file overwrite operations should be enabled + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @throws IOException IO problems + */ + public static void writeDataset(FileSystem fs, + Path path, + byte[] src, + int len, + int buffersize, + boolean overwrite) throws IOException { + assertTrue( + "Not enough data in source array to write " + len + " bytes", + src.length >= len); + FSDataOutputStream out = fs.create(path, + overwrite, + fs.getConf() + .getInt(IO_FILE_BUFFER_SIZE, + 4096), + (short) 1, + buffersize); + out.write(src, 0, len); + out.close(); + assertFileHasLength(fs, path, len); + } + + /** + * Read the file and convert to a byte dataset. + * This implements readfully internally, so that it will read + * in the file without ever having to seek() + * @param fs filesystem + * @param path path to read from + * @param len length of data to read + * @return the bytes + * @throws IOException IO problems + */ + public static byte[] readDataset(FileSystem fs, Path path, int len) + throws IOException { + FSDataInputStream in = fs.open(path); + byte[] dest = new byte[len]; + int offset =0; + int nread = 0; + try { + while (nread < len) { + int nbytes = in.read(dest, offset + nread, len - nread); + if (nbytes < 0) { + throw new EOFException("End of file reached before reading fully."); + } + nread += nbytes; + } + } finally { + in.close(); + } + return dest; + } + + /** + * Read a file, verify its length and contents match the expected array + * @param fs filesystem + * @param path path to file + * @param original original dataset + * @throws IOException IO Problems + */ + public static void verifyFileContents(FileSystem fs, + Path path, + byte[] original) throws IOException { + FileStatus stat = fs.getFileStatus(path); + String statText = stat.toString(); + assertTrue("not a file " + statText, stat.isFile()); + assertEquals("wrong length " + statText, original.length, stat.getLen()); + byte[] bytes = readDataset(fs, path, original.length); + compareByteArrays(original,bytes,original.length); + } + + /** + * Verify that the read at a specific offset in a stream + * matches that expected + * @param stm stream + * @param fileContents original file contents + * @param seekOff seek offset + * @param toRead number of bytes to read + * @throws IOException IO problems + */ + public static void verifyRead(FSDataInputStream stm, byte[] fileContents, + int seekOff, int toRead) throws IOException { + byte[] out = new byte[toRead]; + stm.seek(seekOff); + stm.readFully(out); + byte[] expected = Arrays.copyOfRange(fileContents, seekOff, + seekOff + toRead); + compareByteArrays(expected, out,toRead); + } + + /** + * Assert that tthe array original[0..len] and received[] are equal. + * A failure triggers the logging of the bytes near where the first + * difference surfaces. + * @param original source data + * @param received actual + * @param len length of bytes to compare + */ + public static void compareByteArrays(byte[] original, + byte[] received, + int len) { + assertEquals("Number of bytes read != number written", + len, received.length); + int errors = 0; + int first_error_byte = -1; + for (int i = 0; i < len; i++) { + if (original[i] != received[i]) { + if (errors == 0) { + first_error_byte = i; + } + errors++; + } + } + + if (errors > 0) { + String message = String.format(" %d errors in file of length %d", + errors, len); + LOG.warn(message); + // the range either side of the first error to print + // this is a purely arbitrary number, to aid user debugging + final int overlap = 10; + for (int i = Math.max(0, first_error_byte - overlap); + i < Math.min(first_error_byte + overlap, len); + i++) { + byte actual = received[i]; + byte expected = original[i]; + String letter = toChar(actual); + String line = String.format("[%04d] %2x %s\n", i, actual, letter); + if (expected != actual) { + line = String.format("[%04d] %2x %s -expected %2x %s\n", + i, + actual, + letter, + expected, + toChar(expected)); + } + LOG.warn(line); + } + fail(message); + } + } + + /** + * Convert a byte to a character for printing. If the + * byte value is < 32 -and hence unprintable- the byte is + * returned as a two digit hex value + * @param b byte + * @return the printable character string + */ + public static String toChar(byte b) { + if (b >= 0x20) { + return Character.toString((char) b); + } else { + return String.format("%02x", b); + } + } + + /** + * Convert a buffer to a string, character by character + * @param buffer input bytes + * @return a string conversion + */ + public static String toChar(byte[] buffer) { + StringBuilder builder = new StringBuilder(buffer.length); + for (byte b : buffer) { + builder.append(toChar(b)); + } + return builder.toString(); + } + + public static byte[] toAsciiByteArray(String s) { + char[] chars = s.toCharArray(); + int len = chars.length; + byte[] buffer = new byte[len]; + for (int i = 0; i < len; i++) { + buffer[i] = (byte) (chars[i] & 0xff); + } + return buffer; + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param cleanupPath path to delete as a string + */ + public static void cleanup(String action, + FileSystem fileSystem, + String cleanupPath) { + if (fileSystem == null) { + return; + } + Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(), + fileSystem.getWorkingDirectory()); + cleanup(action, fileSystem, path); + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + */ + public static void cleanup(String action, FileSystem fileSystem, Path path) { + noteAction(action); + try { + rm(fileSystem, path, true, false); + } catch (Exception e) { + LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e); + } + } + + /** + * Delete a directory. There's a safety check for operations against the + * root directory -these are intercepted and rejected with an IOException + * unless the allowRootDelete flag is true + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + * @param recursive flag to enable recursive delete + * @param allowRootDelete can the root directory be deleted? + * @throws IOException on any problem. + */ + public static boolean rm(FileSystem fileSystem, + Path path, + boolean recursive, + boolean allowRootDelete) throws + IOException { + if (fileSystem != null) { + rejectRootOperation(path, allowRootDelete); + if (fileSystem.exists(path)) { + return fileSystem.delete(path, recursive); + } + } + return false; + + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @param allowRootOperation can the root directory be manipulated? + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path, + boolean allowRootOperation) throws IOException { + if (path.isRoot() && !allowRootOperation) { + throw new IOException("Root directory operation rejected: " + path); + } + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path) throws IOException { + rejectRootOperation(path, false); + } + + + public static void noteAction(String action) { + if (LOG.isDebugEnabled()) { + LOG.debug("============== "+ action +" ============="); + } + } + + /** + * downgrade a failure to a message and a warning, then an + * exception for the Junit test runner to mark as failed + * @param message text message + * @param failure what failed + * @throws AssumptionViolatedException always + */ + public static void downgrade(String message, Throwable failure) { + LOG.warn("Downgrading test " + message, failure); + AssumptionViolatedException ave = + new AssumptionViolatedException(failure, null); + throw ave; + } + + /** + * report an overridden test as unsupported + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void unsupported(String message) { + skip(message); + } + + /** + * report a test has been skipped for some reason + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void skip(String message) { + LOG.info("Skipping: {}", message); + throw new AssumptionViolatedException(message); + } + + /** + * Fail with an exception that was received + * @param text text to use in the exception + * @param thrown a (possibly null) throwable to init the cause with + * @throws AssertionError with the text and throwable -always + */ + public static void fail(String text, Throwable thrown) { + AssertionError e = new AssertionError(text); + e.initCause(thrown); + throw e; + } + + /** + * Make an assertion about the length of a file + * @param fs filesystem + * @param path path of the file + * @param expected expected length + * @throws IOException on File IO problems + */ + public static void assertFileHasLength(FileSystem fs, Path path, + int expected) throws IOException { + FileStatus status = fs.getFileStatus(path); + assertEquals( + "Wrong file length of file " + path + " status: " + status, + expected, + status.getLen()); + } + + /** + * Assert that a path refers to a directory + * @param fs filesystem + * @param path path of the directory + * @throws IOException on File IO problems + */ + public static void assertIsDirectory(FileSystem fs, + Path path) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertIsDirectory(fileStatus); + } + + /** + * Assert that a path refers to a directory + * @param fileStatus stats to check + */ + public static void assertIsDirectory(FileStatus fileStatus) { + assertTrue("Should be a directory -but isn't: " + fileStatus, + fileStatus.isDirectory()); + } + + /** + * Write the text to a file, returning the converted byte array + * for use in validating the round trip + * @param fs filesystem + * @param path path of file + * @param text text to write + * @param overwrite should the operation overwrite any existing file? + * @return the read bytes + * @throws IOException on IO problems + */ + public static byte[] writeTextFile(FileSystem fs, + Path path, + String text, + boolean overwrite) throws IOException { + byte[] bytes = new byte[0]; + if (text != null) { + bytes = toAsciiByteArray(text); + } + createFile(fs, path, overwrite, bytes); + return bytes; + } + + /** + * Create a file + * @param fs filesystem + * @param path path to write + * @param overwrite overwrite flag + * @param data source dataset. Can be null + * @throws IOException on any problem + */ + public static void createFile(FileSystem fs, + Path path, + boolean overwrite, + byte[] data) throws IOException { + FSDataOutputStream stream = fs.create(path, overwrite); + if (data != null && data.length > 0) { + stream.write(data); + } + stream.close(); + } + + /** + * Touch a file + * @param fs filesystem + * @param path path + * @throws IOException IO problems + */ + public static void touch(FileSystem fs, + Path path) throws IOException { + createFile(fs, path, true, null); + } + + /** + * Delete a file/dir and assert that delete() returned true + * <i>and</i> that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive) throws IOException { + assertDeleted(fs, file, recursive, false); + } + + /** + * Delete a file/dir and assert that delete() returned true + * <i>and</i> that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @param allowRootOperations can the root dir be deleted? + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive, + boolean allowRootOperations) throws IOException { + rejectRootOperation(file, allowRootOperations); + assertPathExists(fs, "about to be deleted file", file); + boolean deleted = fs.delete(file, recursive); + String dir = ls(fs, file.getParent()); + assertTrue("Delete failed on " + file + ": " + dir, deleted); + assertPathDoesNotExist(fs, "Deleted file", file); + } + + /** + * Read in "length" bytes, convert to an ascii string + * @param fs filesystem + * @param path path to read + * @param length #of bytes to read. + * @return the bytes read and converted to a string + * @throws IOException IO problems + */ + public static String readBytesToString(FileSystem fs, + Path path, + int length) throws IOException { + FSDataInputStream in = fs.open(path); + try { + byte[] buf = new byte[length]; + in.readFully(0, buf); + return toChar(buf); + } finally { + in.close(); + } + } + + /** + * Take an array of filestats and convert to a string (prefixed w/ a [01] counter + * @param stats array of stats + * @param separator separator after every entry + * @return a stringified set + */ + public static String fileStatsToString(FileStatus[] stats, String separator) { + StringBuilder buf = new StringBuilder(stats.length * 128); + for (int i = 0; i < stats.length; i++) { + buf.append(String.format("[%02d] %s", i, stats[i])).append(separator); + } + return buf.toString(); + } + + /** + * List a directory + * @param fileSystem FS + * @param path path + * @return a directory listing or failure message + * @throws IOException + */ + public static String ls(FileSystem fileSystem, Path path) throws IOException { + if (path == null) { + //surfaces when someone calls getParent() on something at the top of the path + return "/"; + } + FileStatus[] stats; + String pathtext = "ls " + path; + try { + stats = fileSystem.listStatus(path); + } catch (FileNotFoundException e) { + return pathtext + " -file not found"; + } catch (IOException e) { + return pathtext + " -failed: " + e; + } + return dumpStats(pathtext, stats); + } + + public static String dumpStats(String pathname, FileStatus[] stats) { + return pathname + fileStatsToString(stats, "\n"); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param fileSystem filesystem to resolve path against + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + public static void assertIsFile(FileSystem fileSystem, Path filename) throws + IOException { + assertPathExists(fileSystem, "Expected file", filename); + FileStatus status = fileSystem.getFileStatus(filename); + assertIsFile(filename, status); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param filename name of the file + * @param status file status + */ + public static void assertIsFile(Path filename, FileStatus status) { + String fileInfo = filename + " " + status; + assertFalse("File claims to be a directory " + fileInfo, + status.isDirectory()); + assertFalse("File claims to be a symlink " + fileInfo, + status.isSymlink()); + } + + /** + * Create a dataset for use in the tests; all data is in the range + * base to (base+modulo-1) inclusive + * @param len length of data + * @param base base of the data + * @param modulo the modulo + * @return the newly generated dataset + */ + public static byte[] dataset(int len, int base, int modulo) { + byte[] dataset = new byte[len]; + for (int i = 0; i < len; i++) { + dataset[i] = (byte) (base + (i % modulo)); + } + return dataset; + } + + /** + * Assert that a path exists -but make no assertions as to the + * type of that entry + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws FileNotFoundException raised if the path is missing + * @throws IOException IO problems + */ + public static void assertPathExists(FileSystem fileSystem, String message, + Path path) throws IOException { + if (!fileSystem.exists(path)) { + //failure, report it + String listing = ls(fileSystem, path.getParent()); + throw new FileNotFoundException(message + ": not found " + path + + " in \"" + path.getParent() + "\" :\n" + listing); + } + } + + /** + * Assert that a path does not exist + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist(FileSystem fileSystem, + String message, + Path path) throws IOException { + try { + FileStatus status = fileSystem.getFileStatus(path); + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** + * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry + * @param fs filesystem + * @param dir directory to scan + * @param subdir full path to look for + * @throws IOException IO probles + */ + public static void assertListStatusFinds(FileSystem fs, + Path dir, + Path subdir) throws IOException { + FileStatus[] stats = fs.listStatus(dir); + boolean found = false; + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.toString()).append('\n'); + if (stat.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + /** + * Test for the host being an OSX machine + * @return true if the JVM thinks that is running on OSX + */ + public static boolean isOSX() { + return System.getProperty("os.name").contains("OS X"); + } + + /** + * compare content of file operations using a double byte array + * @param concat concatenated files + * @param bytes bytes + */ + public static void validateFileContent(byte[] concat, byte[][] bytes) { + int idx = 0; + boolean mismatch = false; + + for (byte[] bb : bytes) { + for (byte b : bb) { + if (b != concat[idx++]) { + mismatch = true; + break; + } + } + if (mismatch) + break; + } + assertFalse("File content of file is not as expected at offset " + idx, + mismatch); + } + + /** + * Receives test data from the given input file and checks the size of the + * data as well as the pattern inside the received data. + * + * @param fs FileSystem + * @param path Input file to be checked + * @param expectedSize the expected size of the data to be read from the + * input file in bytes + * @param bufferLen Pattern length + * @param modulus Pattern modulus + * @throws IOException + * thrown if an error occurs while reading the data + */ + public static void verifyReceivedData(FileSystem fs, Path path, + final long expectedSize, + final int bufferLen, + final int modulus) throws IOException { + final byte[] testBuffer = new byte[bufferLen]; + + long totalBytesRead = 0; + int nextExpectedNumber = 0; + final InputStream inputStream = fs.open(path); + try { + while (true) { + final int bytesRead = inputStream.read(testBuffer); + if (bytesRead < 0) { + break; + } + + totalBytesRead += bytesRead; + + for (int i = 0; i < bytesRead; ++i) { + if (testBuffer[i] != nextExpectedNumber) { + throw new IOException("Read number " + testBuffer[i] + + " but expected " + nextExpectedNumber); + } + + ++nextExpectedNumber; + + if (nextExpectedNumber == modulus) { + nextExpectedNumber = 0; + } + } + } + + if (totalBytesRead != expectedSize) { + throw new IOException("Expected to read " + expectedSize + + " bytes but only received " + totalBytesRead); + } + } finally { + inputStream.close(); + } + } + + /** + * Generates test data of the given size according to some specific pattern + * and writes it to the provided output file. + * + * @param fs FileSystem + * @param path Test file to be generated + * @param size The size of the test data to be generated in bytes + * @param bufferLen Pattern length + * @param modulus Pattern modulus + * @throws IOException + * thrown if an error occurs while writing the data + */ + public static long generateTestFile(FileSystem fs, Path path, + final long size, + final int bufferLen, + final int modulus) throws IOException { + final byte[] testBuffer = new byte[bufferLen]; + for (int i = 0; i < testBuffer.length; ++i) { + testBuffer[i] = (byte) (i % modulus); + } + + final OutputStream outputStream = fs.create(path, false); + long bytesWritten = 0; + try { + while (bytesWritten < size) { + final long diff = size - bytesWritten; + if (diff < testBuffer.length) { + outputStream.write(testBuffer, 0, (int) diff); + bytesWritten += diff; + } else { + outputStream.write(testBuffer); + bytesWritten += testBuffer.length; + } + } + + return bytesWritten; + } finally { + outputStream.close(); + } + } + + /** + * Creates and reads a file with the given size. The test file is generated + * according to a specific pattern so it can be easily verified even if it's + * a multi-GB one. + * During the read phase the incoming data stream is also checked against + * this pattern. + * + * @param fs FileSystem + * @param parent Test file parent dir path + * @throws IOException + * thrown if an I/O error occurs while writing or reading the test file + */ + public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize) + throws IOException { + int testBufferSize = fs.getConf() + .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE); + int modulus = fs.getConf() + .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE); + + final String objectName = UUID.randomUUID().toString(); + final Path objectPath = new Path(parent, objectName); + + // Write test file in a specific pattern + assertEquals(fileSize, + generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus)); + assertPathExists(fs, "not created successful", objectPath); + + // Now read the same file back and verify its content + try { + verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus); + } finally { + // Delete test file + fs.delete(objectPath, false); + } + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java new file mode 100644 index 0000000..cf96407 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXMLConfKeysForTesting; + +/** + * Keys shared across tests. + */ +public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting { + /** + * Username for all clusters, ZK, etc. + */ + String USERNAME = "bigdataborat"; + + int WAIT_TIME = 120; + String WAIT_TIME_ARG = Integer.toString(WAIT_TIME); + + String SLIDER_TEST_XML = "slider-test.xml"; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java new file mode 100644 index 0000000..be452f1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService; +import org.apache.hadoop.registry.server.services.MicroZookeeperService; +import org.apache.slider.common.tools.SliderUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; + +/** + * Test ZK cluster. + */ +public class MicroZKCluster implements Closeable { + private static final Logger LOG = + LoggerFactory.getLogger(MicroZKCluster.class); + + public static final String HOSTS = "127.0.0.1"; + private MicroZookeeperService zkService; + private String zkBindingString; + private final Configuration conf; + private RegistryOperations registryOperations; + + MicroZKCluster() { + this(SliderUtils.createConfiguration()); + } + + MicroZKCluster(Configuration conf) { + this.conf = conf; + } + + String getZkBindingString() { + return zkBindingString; + } + + void createCluster(String name) { + zkService = new MicroZookeeperService(name); + + zkService.init(conf); + zkService.start(); + zkBindingString = zkService.getConnectionString(); + LOG.info("Created {}", this); + registryOperations = new RegistryOperationsService( + "registry", + zkService); + registryOperations.init(conf); + registryOperations.start(); + } + + @Override + public void close() throws IOException { + if (registryOperations != null) { + registryOperations.stop(); + } + if (zkService != null) { + zkService.stop(); + } + } + + @Override + public String toString() { + return "Micro ZK cluster as " + zkBindingString; + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java new file mode 100644 index 0000000..52875d3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +/** + * Outcome for probes. + */ +public final class Outcome { + + private final String name; + + private Outcome(String name) { + this.name = name; + } + + public static final Outcome SUCCESS = new Outcome( + "Success"); + public static final Outcome RETRY = new Outcome("Retry"); + public static final Outcome FAIL = new Outcome("Fail"); + + /** + * Build from a bool, where false is mapped to retry. + * @param b boolean + * @return an outcome + */ + static Outcome fromBool(boolean b) { + return b ? SUCCESS : RETRY; + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java new file mode 100644 index 0000000..f7da585 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +import org.apache.hadoop.fs.FileUtil; +import org.apache.slider.common.SliderXMLConfKeysForTesting; +import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; + +import java.io.File; + + +/** + * Base class for unit tests as well as ones starting mini clusters + * -the foundational code and methods. + * + */ +public abstract class SliderTestBase extends SliderTestUtils { + + /** + * Singleton metric registry. + */ + public static final MetricsAndMonitoring METRICS = new MetricsAndMonitoring(); + public static final int WEB_STARTUP_TIME = 30000; + + @Rule + public TestName methodName = new TestName(); + + @BeforeClass + public static void nameThread() { + Thread.currentThread().setName("JUnit"); + } + + @Before + public void setup() throws Exception { + setSliderClientClassName(DEFAULT_SLIDER_CLIENT); + FileUtil.fullyDelete(new File(SliderXMLConfKeysForTesting + .TEST_SECURITY_DIR)); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java new file mode 100644 index 0000000..fc29b5e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java @@ -0,0 +1,1065 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParser; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.service.ServiceStateException; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Container; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.tools.Duration; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.main.LauncherExitCodes; +import org.apache.slider.core.main.ServiceLaunchException; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.core.persist.JsonSerDeser; +import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; +import org.apache.slider.server.services.workflow.ForkedProcessService; +import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.junit.Assert; +import org.junit.Assume; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeoutException; + +import static org.apache.slider.common.params.Arguments.ARG_OPTION; + +/** + * Static utils for tests in this package and in other test projects. + * + * It is designed to work with mini clusters as well as remote ones + * + * This class is not final and may be extended for test cases. + * + * Some of these methods are derived from the SwiftUtils and SwiftTestUtils + * classes -replicated here so that they are available in Hadoop-2.0 code + */ +public class SliderTestUtils extends Assert { + private static final Logger LOG = + LoggerFactory.getLogger(SliderTestUtils.class); + public static final String DEFAULT_SLIDER_CLIENT = SliderClient.class + .getName(); + private static String sliderClientClassName = DEFAULT_SLIDER_CLIENT; + + public static final Map<String, String> EMPTY_MAP = Collections.emptyMap(); + public static final Map<String, Integer> EMPTY_INT_MAP = Collections + .emptyMap(); + public static final List<String> EMPTY_LIST = Collections.emptyList(); + + public static final ObjectReader OBJECT_READER; + public static final ObjectWriter OBJECT_WRITER; + + public static final JsonSerDeser<Application> JSON_SER_DESER = + new JsonSerDeser<>(Application.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + + static { + ObjectMapper mapper = new ObjectMapper(); + OBJECT_READER = mapper.readerFor(Object.class); + OBJECT_WRITER = mapper.writer(); + } + + /** + * Action that returns an object. + */ + public interface Action { + Object invoke() throws Exception; + } + + /** + * Probe that returns an Outcome. + */ + public interface Probe { + Outcome invoke(Map args) throws Exception; + } + + public static void setSliderClientClassName(String sliderClientClassName) { + sliderClientClassName = sliderClientClassName; + } + + public static void describe(String s) { + LOG.info(""); + LOG.info("==============================="); + LOG.info(s); + LOG.info("==============================="); + LOG.info(""); + } + + /** + * Convert a JSON string to something readable. + * @param json + * @return a string for printing + */ + public static String prettyPrintJson(String json) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + return gson.toJson(new JsonParser().parse(json)); + } + + /** + * Convert an object to something readable. + * @param src + * @return a string for printing + */ + public static String prettyPrintAsJson(Object src) + throws JsonProcessingException, UnsupportedEncodingException { + return new String(OBJECT_WRITER.writeValueAsBytes(src), "UTF8"); + } + + /** + * Skip the test with a message. + * @param message message logged and thrown + */ + public static void skip(String message) { + LOG.warn("Skipping test: {}", message); + Assume.assumeTrue(message, false); + } + + /** + * Skip the test with a message if condition holds. + * @param condition predicate + * @param message message logged and thrown + */ + public static void assume(boolean condition, String message) { + if (!condition) { + skip(message); + } + } + + /** + * Skip a test if not running on Windows. + */ + public static void assumeWindows() { + assume(Shell.WINDOWS, "not windows"); + } + + /** + * Skip a test if running on Windows. + */ + public static void assumeNotWindows() { + assume(!Shell.WINDOWS, "windows"); + } + + /** + * Skip a test on windows. + */ + public static void skipOnWindows() { + assumeNotWindows(); + } + + /** + * Equality size for a list. + * @param left + * @param right + */ + public static void assertListEquals(List left, List right) { + String lval = collectionToString(left); + String rval = collectionToString(right); + String text = "comparing " + lval + " to " + rval; + assertEquals(text, left.size(), right.size()); + for (int i = 0; i < left.size(); i++) { + assertEquals(text, left.get(i), right.get(i)); + } + } + + /** + * Assert a list has a given length. + * @param list list + * @param size size to have + */ + public static void assertListLength(List list, int size) { + String lval = collectionToString(list); + assertEquals(lval, size, list.size()); + } + + /** + * Stringify a collection with [ ] at either end. + * @param collection collection + * @return string value + */ + public static String collectionToString(List collection) { + return "[" + SliderUtils.join(collection, ", ", false) + "]"; + } + + /** + * Assume that a string option is set and not equal to "". + * @param conf configuration file + * @param key key to look for + */ + public static void assumeStringOptionSet(Configuration conf, String key) { + if (SliderUtils.isUnset(conf.getTrimmed(key))) { + skip("Configuration key " + key + " not set"); + } + } + + /** + * assert that a string option is set and not equal to "". + * @param conf configuration file + * @param key key to look for + */ + public static void assertStringOptionSet(Configuration conf, String key) { + getRequiredConfOption(conf, key); + } + + /** + * Assume that a boolean option is set and true. + * Unset or false triggers a test skip + * @param conf configuration file + * @param key key to look for + */ + public static void assumeBoolOptionTrue(Configuration conf, String key) { + assumeBoolOption(conf, key, false); + } + + /** + * Assume that a boolean option is true. + * False triggers a test skip + * @param conf configuration file + * @param key key to look for + * @param defval default value if the property is not defined + */ + public static void assumeBoolOption( + Configuration conf, String key, boolean defval) { + assume(conf.getBoolean(key, defval), + "Configuration key " + key + " is false"); + } + + /** + * Get a required config option (trimmed, incidentally). + * Test will fail if not set + * @param conf configuration + * @param key key + * @return the string + */ + public static String getRequiredConfOption(Configuration conf, String key) { + String val = conf.getTrimmed(key); + if (SliderUtils.isUnset(val)) { + fail("Missing configuration option " + key); + } + return val; + } + + /** + * Fails a test because required behavior has not been implemented. + */ + public static void failNotImplemented() { + fail("Not implemented"); + } + + /** + * Assert that any needed libraries being present. On Unix none are needed; + * on windows they must be present + */ + public static void assertNativeLibrariesPresent() { + String errorText = SliderUtils.checkForRequiredNativeLibraries(); + if (SliderUtils.isSet(errorText)) { + fail(errorText); + } + } + + protected static String[] toArray(List<Object> args) { + String[] converted = new String[args.size()]; + for (int i = 0; i < args.size(); i++) { + Object elt = args.get(i); + assertNotNull(args.get(i)); + converted[i] = elt.toString(); + } + return converted; + } + + public static void waitWhileClusterLive(SliderClient client, int timeout) + throws IOException, YarnException { + Duration duration = new Duration(timeout); + duration.start(); + while (client.actionExists(client.getDeployedClusterName(), true) == + LauncherExitCodes.EXIT_SUCCESS && !duration.getLimitExceeded()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + if (duration.getLimitExceeded()) { + fail("Cluster " + client.getDeployedClusterName() + " still live after " + + timeout + " ms"); + } + } + + public static void waitUntilClusterLive(SliderClient client, int timeout) + throws IOException, YarnException { + Duration duration = new Duration(timeout); + duration.start(); + while (LauncherExitCodes.EXIT_SUCCESS != client.actionExists( + client.getDeployedClusterName(), true) && + !duration.getLimitExceeded()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + } + if (duration.getLimitExceeded()) { + fail("Cluster " + client.getDeployedClusterName() + " not live after " + + timeout + " ms"); + } + } + + public static void dumpClusterDescription( + String text, + Application status) throws IOException { + describe(text); + LOG.info(JSON_SER_DESER.toJson(status)); + } + + /** + * Assert that a service operation succeeded. + * @param service service + */ + public static void assertSucceeded(ServiceLauncher service) { + assertEquals(0, service.getServiceExitCode()); + } + + public static void assertContainersLive(Application application, + String component, int expected) { + LOG.info("Asserting component {} expected count {}", component, expected); + int actual = extractLiveContainerCount(application, component); + if (expected != actual) { + LOG.warn("{} actual={}, expected {} in \n{}\n", component, actual, + expected, application); + } + assertEquals(expected, actual); + } + + /** + * Robust extraction of live container count. + * @param application status + * @param component component to resolve + * @return the number of containers live. + */ + public static int extractLiveContainerCount( + Application application, + String component) { + int actual = 0; + if (application.getContainers() != null) { + for (Container container : application.getContainers()) { + if (container.getComponentName().equals(component)) { + actual++; + } + } + } + return actual; + } + + /** + * Exec a set of commands, wait a few seconds for it to finish. + * @param status code + * @param commands + * @return the process + */ + public static ForkedProcessService exec(int status, List<String> commands) + throws IOException, TimeoutException { + ForkedProcessService process = exec(commands); + + Integer exitCode = process.getExitCode(); + assertNotNull(exitCode); + assertEquals(status, exitCode.intValue()); + return process; + } + + /** + * Exec a set of commands, wait a few seconds for it to finish. + * @param commands + * @return + */ + public static ForkedProcessService exec(List<String> commands) + throws IOException, TimeoutException { + ForkedProcessService process; + process = new ForkedProcessService( + commands.get(0), + EMPTY_MAP, + commands); + process.init(new Configuration()); + process.start(); + int timeoutMillis = 5000; + if (!process.waitForServiceToStop(timeoutMillis)) { + throw new TimeoutException( + "Process did not stop in " + timeoutMillis + "mS"); + } + return process; + } + + /** + * Determine whether an application exists. Run the commands and if the + * operation fails with a FileNotFoundException, then + * this method returns false. + * <p> + * Run something harmless like a -version command, something + * which must return 0 + * + * @param commands + * @return true if the command sequence succeeded + * false if they failed with no file + * @throws Exception on any other failure cause + */ + public static boolean doesAppExist(List<String> commands) + throws IOException, TimeoutException { + try { + exec(0, commands); + return true; + } catch (ServiceStateException e) { + if (!(e.getCause() instanceof FileNotFoundException)) { + throw e; + } + return false; + } + } + + /** + * Locate an executable on the path. + * @param exe executable name. If it is an absolute path which + * exists then it will returned direct + * @return the path to an exe or null for no match + */ + public static File locateExecutable(String exe) { + File exeNameAsPath = new File(exe).getAbsoluteFile(); + if (exeNameAsPath.exists()) { + return exeNameAsPath; + } + + File exepath = null; + String path = extractPath(); + String[] dirs = path.split(System.getProperty("path.separator")); + for (String dirname : dirs) { + File dir = new File(dirname); + + File possible = new File(dir, exe); + if (possible.exists()) { + exepath = possible; + } + } + return exepath; + } + + /** + * Lookup the PATH env var. + * @return the path or null + */ + public static String extractPath() { + return extractEnvVar("PATH"); + } + + /** + * Find an environment variable. Uses case independent checking for + * the benefit of windows. + * Will fail if the var is not found. + * @param var path variable <i>in upper case</i> + * @return the env var + */ + public static String extractEnvVar(String var) { + String realkey = ""; + + for (String it : System.getenv().keySet()) { + if (it.toUpperCase(Locale.ENGLISH).equals(var)) { + realkey = it; + } + } + + if (SliderUtils.isUnset(realkey)) { + fail("No environment variable " + var + " found"); + } + String val = System.getenv(realkey); + + LOG.info("{} = {}", realkey, val); + return val; + } + + /** + * Create a temp JSON file. After coming up with the name, the file + * is deleted + * @return the filename + */ + public static File createTempJsonFile() throws IOException { + return tmpFile(".json"); + } + + /** + * Create a temp file with the specific name. It's deleted after creation, + * to avoid "file exists exceptions" + * @param suffix suffix, e.g. ".txt" + * @return a path to a file which may be created + */ + public static File tmpFile(String suffix) throws IOException { + File reportFile = File.createTempFile( + "temp", + suffix, + new File("target")); + reportFile.delete(); + return reportFile; + } + + /** + * Execute a closure, assert it fails with a given exit code and text. + * @param exitCode exit code + * @param text text (can be "") + * @param action action + * @return + */ + public void assertFailsWithException(int exitCode, + String text, + Action action) throws Exception { + try { + action.invoke(); + fail("Operation was expected to fail âbut it succeeded"); + } catch (ServiceLaunchException e) { + assertExceptionDetails(e, exitCode, text); + } + } + + /** + * Execute a closure, assert it fails with a given exit code and text. + * @param text text (can be "") + * @param action action + * @return + */ + public void assertFailsWithExceptionClass(Class clazz, + String text, + Action action) throws Exception { + try { + action.invoke(); + fail("Operation was expected to fail âbut it succeeded"); + } catch (Exception e) { + assertExceptionDetails(e, clazz, text); + } + } + + public static void assertExceptionDetails( + ServiceLaunchException ex, + int exitCode) { + assertExceptionDetails(ex, exitCode, null); + } + + /** + * Make an assertion about the exit code of an exception. + * @param ex exception + * @param exitCode exit code + * @param text error text to look for in the exception + */ + public static void assertExceptionDetails( + ServiceLaunchException ex, + int exitCode, + String text) { + if (exitCode != ex.getExitCode()) { + String message = String.format("Wrong exit code, expected %d but" + + " got %d in %s", exitCode, ex.getExitCode(), ex); + LOG.warn(message, ex); + throw new AssertionError(message, ex); + } + if (SliderUtils.isSet(text)) { + if (!(ex.toString().contains(text))) { + String message = String.format("String match for \"%s\"failed in %s", + text, ex); + LOG.warn(message, ex); + throw new AssertionError(message, ex); + } + } + } + + /** + * Make an assertion about the class of an exception. + * @param ex exception + * @param clazz exit code + * @param text error text to look for in the exception + */ + static void assertExceptionDetails( + Exception ex, + Class clazz, + String text) throws Exception { + if (ex.getClass() != clazz) { + throw ex; + } + if (SliderUtils.isSet(text) && !(ex.toString().contains(text))) { + throw ex; + } + } + + /** + * Launch the slider client with the specific args; no validation + * of return code takes place. + * @param conf configuration + * @param args arg list + * @return the launcher + */ + protected static ServiceLauncher<SliderClient> execSliderCommand( + Configuration conf, + List args) throws Throwable { + ServiceLauncher<SliderClient> serviceLauncher = + new ServiceLauncher<>(sliderClientClassName); + + LOG.debug("slider {}", SliderUtils.join(args, " ", false)); + serviceLauncher.launchService(conf, + toArray(args), + false); + return serviceLauncher; + } + + /** + * Launch a slider command to a given exit code. + * Most failures will trigger exceptions; this is for the exit code of the + * runService() call. + * @param exitCode desired exit code + * @param conf configuration + * @param args arg list + * @return the launcher + */ + protected static ServiceLauncher<SliderClient> execSliderCommand( + int exitCode, + Configuration conf, + List args) throws Throwable { + ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf, + args); + assertEquals(exitCode, serviceLauncher.getServiceExitCode()); + return serviceLauncher; + } + + public static ServiceLauncher launch(Class serviceClass, + Configuration conf, + List<Object> args) throws + Throwable { + ServiceLauncher serviceLauncher = + new ServiceLauncher(serviceClass.getName()); + + String joinedArgs = SliderUtils.join(args, " ", false); + LOG.debug("slider {}", joinedArgs); + + serviceLauncher.launchService(conf, + toArray(args), + false); + return serviceLauncher; + } + + public static Throwable launchExpectingException(Class serviceClass, + Configuration conf, + String expectedText, + List args) + throws Throwable { + try { + ServiceLauncher launch = launch(serviceClass, conf, args); + throw new AssertionError("Expected an exception with text containing " + + expectedText + " -but the service completed with exit code " + + launch.getServiceExitCode()); + } catch (AssertionError error) { + throw error; + } catch (Throwable thrown) { + if (SliderUtils.isSet(expectedText) && !thrown.toString().contains( + expectedText)) { + //not the right exception -rethrow + LOG.warn("Caught Exception did not contain expected text" + + "\"" + expectedText + "\""); + throw thrown; + } + return thrown; + } + } + + + public static ServiceLauncher<SliderClient> launchClientAgainstRM( + String address, + List<String> args, + Configuration conf) throws Throwable { + assertNotNull(address); + LOG.info("Connecting to rm at {}", address); + if (!args.contains(Arguments.ARG_MANAGER)) { + args.add(Arguments.ARG_MANAGER); + args.add(address); + } + ServiceLauncher<SliderClient> launcher = execSliderCommand(conf, args); + return launcher; + } + + /** + * Add a configuration parameter as a cluster configuration option. + * @param extraArgs extra arguments + * @param conf config + * @param option option + */ + public static void addClusterConfigOption( + List<String> extraArgs, + YarnConfiguration conf, + String option) { + + conf.getTrimmed(option); + extraArgs.add(ARG_OPTION); + extraArgs.add(option); + extraArgs.add(getRequiredConfOption(conf, option)); + } + + /** + * Assert that a path refers to a directory. + * @param fs filesystem + * @param path path of the directory + * @throws IOException on File IO problems + */ + public static void assertIsDirectory(FileSystem fs, + Path path) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertIsDirectory(fileStatus); + } + + /** + * Assert that a path refers to a directory. + * @param fileStatus stats to check + */ + public static void assertIsDirectory(FileStatus fileStatus) { + assertTrue("Should be a dir -but isn't: " + fileStatus, + fileStatus.isDirectory()); + } + + /** + * Assert that a path exists -but make no assertions as to the + * type of that entry. + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathExists( + FileSystem fileSystem, + String message, + Path path) throws IOException { + if (!fileSystem.exists(path)) { + //failure, report it + fail( + message + ": not found \"" + path + "\" in " + path.getParent() + + "-" + + ls(fileSystem, path.getParent())); + } + } + + /** + * Assert that a path does not exist. + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist( + FileSystem fileSystem, + String message, + Path path) throws IOException { + try { + FileStatus status = fileSystem.getFileStatus(path); + // a status back implies there is a file here + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** + * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry. + * @param fs filesystem + * @param dir directory to scan + * @param subdir full path to look for + * @throws IOException IO probles + */ + public static void assertListStatusFinds(FileSystem fs, + Path dir, + Path subdir) throws IOException { + FileStatus[] stats = fs.listStatus(dir); + boolean found = false; + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.toString()).append('\n'); + if (stat.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + /** + * List a a path to string. + * @param fileSystem filesystem + * @param path directory + * @return a listing of the filestatuses of elements in the directory, one + * to a line, precedeed by the full path of the directory + * @throws IOException connectivity problems + */ + public static String ls(FileSystem fileSystem, Path path) + throws IOException { + if (path == null) { + //surfaces when someone calls getParent() on something at the top of + // the path + return "/"; + } + FileStatus[] stats; + String pathtext = "ls " + path; + try { + stats = fileSystem.listStatus(path); + } catch (FileNotFoundException e) { + return pathtext + " -file not found"; + } catch (IOException e) { + return pathtext + " -failed: " + e; + } + return pathtext + fileStatsToString(stats, "\n"); + } + + /** + * Take an array of filestats and convert to a string (prefixed w/ a [01] + * counter). + * @param stats array of stats + * @param separator separator after every entry + * @return a stringified set + */ + public static String fileStatsToString(FileStatus[] stats, String separator) { + StringBuilder buf = new StringBuilder(stats.length * 128); + for (int i = 0; i < stats.length; i++) { + buf.append(String.format("[%02d] %s", i, stats[i])).append(separator); + } + return buf.toString(); + } + + public static void waitWhileClusterLive(SliderClient sliderClient) + throws IOException, YarnException { + waitWhileClusterLive(sliderClient, 30000); + } + + public static void dumpRegistryInstances( + Map<String, ServiceRecord> instances) { + describe("service registry slider instances"); + for (Entry<String, ServiceRecord> it : instances.entrySet()) { + LOG.info(" {} : {}", it.getKey(), it.getValue()); + } + describe("end list service registry slider instances"); + } + + + public static void dumpRegistryInstanceIDs(List<String> instanceIds) { + describe("service registry instance IDs"); + dumpCollection(instanceIds); + } + + public static void dumpRegistryServiceTypes(Collection<String> entries) { + describe("service registry types"); + dumpCollection(entries); + } + + public static <V> void dumpCollection(Collection<V> entries) { + LOG.info("number of entries: {}", entries.size()); + for (V it : entries) { + LOG.info(it.toString()); + } + } + + public static void dumpArray(Object[] entries) { + LOG.info("number of entries: {}", entries.length); + for (Object it : entries) { + LOG.info(it.toString()); + } + } + + public static <K, V> void dumpMap(Map<K, V> map) { + for (Entry<K, V> it : map.entrySet()) { + LOG.info("\"{}\": \"{}\"", it.getKey().toString(), it.getValue() + .toString()); + } + } + + /** + * Get a time option in seconds if set, otherwise the default value (also + * in seconds). + * This operation picks up the time value as a system property if set -that + * value overrides anything in the test file + * @param conf + * @param key + * @param defValMillis + * @return + */ + public static int getTimeOptionMillis( + Configuration conf, + String key, + int defValMillis) { + int val = conf.getInt(key, 0); + val = Integer.getInteger(key, val); + int time = 1000 * val; + if (time == 0) { + time = defValMillis; + } + return time; + } + + public void dumpConfigurationSet(PublishedConfigSet confSet) { + for (String key : confSet.keys()) { + PublishedConfiguration config = confSet.get(key); + LOG.info("{} -- {}", key, config.description); + } + } + + /** + * Convert a file to a URI suitable for use in an argument. + * @param file file + * @return a URI string valid on all platforms + */ + public String toURIArg(File file) { + return file.getAbsoluteFile().toURI().toString(); + } + + /** + * Assert a file exists; fails with a listing of the parent dir. + * @param text text for front of message + * @param file file to look for + * @throws FileNotFoundException + */ + public void assertFileExists(String text, File file) + throws FileNotFoundException { + if (!file.exists()) { + File parent = file.getParentFile(); + String[] files = parent.list(); + StringBuilder builder = new StringBuilder(); + builder.append(parent.getAbsolutePath()); + builder.append(":\n"); + for (String name : files) { + builder.append(" "); + builder.append(name); + builder.append("\n"); + } + throw new FileNotFoundException(text + ": " + file + " not found in " + + builder); + } + } + + /** + * Repeat a probe until it succeeds, if it does not execute a failure + * closure then raise an exception with the supplied message. + * @param probe probe + * @param timeout time in millis before giving up + * @param sleepDur sleep between failing attempts + * @param args map of arguments to the probe + * @param failIfUnsuccessful if the probe fails after all the attempts + * âshould it raise an exception + * @param failureMessage message to include in exception raised + * @param failureHandler closure to invoke prior to the failure being raised + */ + protected void repeatUntilSuccess( + String action, + Probe probe, + int timeout, + int sleepDur, + Map args, + boolean failIfUnsuccessful, + String failureMessage, + Action failureHandler) throws Exception { + LOG.debug("Probe {} timelimit {}", action, timeout); + if (timeout < 1000) { + fail("Timeout " + timeout + " too low: milliseconds are expected, not " + + "seconds"); + } + int attemptCount = 1; + boolean succeeded = false; + boolean completed = false; + Duration duration = new Duration(timeout); + duration.start(); + while (!completed) { + Outcome outcome = probe.invoke(args); + if (outcome.equals(Outcome.SUCCESS)) { + // success + LOG.debug("Success after {} attempt(s)", attemptCount); + succeeded = true; + completed = true; + } else if (outcome.equals(Outcome.RETRY)) { + // failed but retry possible + attemptCount++; + completed = duration.getLimitExceeded(); + if (!completed) { + LOG.debug("Attempt {} failed", attemptCount); + try { + Thread.sleep(sleepDur); + } catch (InterruptedException e) { + } + } + } else if (outcome.equals(Outcome.FAIL)) { + // fast fail + LOG.debug("Fast fail of probe"); + completed = true; + } + } + if (!succeeded) { + if (duration.getLimitExceeded()) { + LOG.info("probe timed out after {} and {} attempts", timeout, + attemptCount); + } + if (failureHandler != null) { + failureHandler.invoke(); + } + if (failIfUnsuccessful) { + fail(failureMessage); + } + } + } + + /** + * Get a value from a map; raise an assertion if it is not there. + * @param map map to look up + * @param key key + * @return the string value + */ + public <K, V> String requiredMapValue(Map<K, V> map, String key) { + assertNotNull(map.get(key)); + return map.get(key).toString(); + } + + public static void assertStringContains(String expected, String text) { + assertNotNull("null text", text); + if (!text.contains(expected)) { + String message = String.format("did not find %s in \"%s\"", expected, + text); + LOG.error(message); + fail(message); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java new file mode 100644 index 0000000..9806ac3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.utils; + +import org.apache.slider.api.resource.Application; +import org.junit.Test; + +import java.util.Collections; + +/** + * Test for some of the command test base operations. + */ +public class TestAssertions { + + public static final String CLUSTER_JSON = "json/cluster.json"; + + @Test + public void testNoInstances() throws Throwable { + Application application = new Application(); + application.setContainers(null); + SliderTestUtils.assertContainersLive(application, "example", 0); + } + + @Test + public void testEmptyInstances() throws Throwable { + Application application = new Application(); + application.setContainers(Collections.emptyList()); + SliderTestUtils.assertContainersLive(application, "example", 0); + } + +// TODO test metrics retrieval +// @Test +// public void testLiveInstances() throws Throwable { +// InputStream stream = getClass().getClassLoader().getResourceAsStream( +// CLUSTER_JSON); +// assertNotNull("could not load " + CLUSTER_JSON, stream); +// ClusterDescription liveCD = ClusterDescription.fromStream(stream); +// assertNotNull(liveCD); +// SliderTestUtils.assertContainersLive(liveCD, "SLEEP_LONG", 4); +// assertEquals((Integer) 1, liveCD.statistics.get("SLEEP_LONG").get( +// StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING)); +// } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java new file mode 100644 index 0000000..5493198 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.utils; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * Various utility methods + * Byte comparison methods are from + * <code>org.apache.hadoop.fs.contract.ContractTestUtils</code> + */ +public class TestUtility { + protected static final Logger log = + LoggerFactory.getLogger(TestUtility.class); + + public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException { + for (File file : dirObj.listFiles()) { + if (file.isDirectory()) { + addDir(file, zipFile, prefix + file.getName() + File.separator); + } else { + log.info("Adding to zip - " + prefix + file.getName()); + zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName())); + IOUtils.copy(new FileInputStream(file), zipFile); + zipFile.closeArchiveEntry(); + } + } + } + + public static void zipDir(String zipFile, String dir) throws IOException { + File dirObj = new File(dir); + ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile)); + log.info("Creating : {}", zipFile); + try { + addDir(dirObj, out, ""); + } finally { + out.close(); + } + } + + public static String createAppPackage( + TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException { + String zipFileName; + File pkgPath = folder.newFolder(subDir); + File zipFile = new File(pkgPath, pkgName).getAbsoluteFile(); + zipFileName = zipFile.getAbsolutePath(); + TestUtility.zipDir(zipFileName, srcPath); + log.info("Created temporary zip file at {}", zipFileName); + return zipFileName; + } + + + /** + * Assert that tthe array original[0..len] and received[] are equal. + * A failure triggers the logging of the bytes near where the first + * difference surfaces. + * @param original source data + * @param received actual + * @param len length of bytes to compare + */ + public static void compareByteArrays(byte[] original, + byte[] received, + int len) { + Assert.assertEquals("Number of bytes read != number written", + len, received.length); + int errors = 0; + int first_error_byte = -1; + for (int i = 0; i < len; i++) { + if (original[i] != received[i]) { + if (errors == 0) { + first_error_byte = i; + } + errors++; + } + } + + if (errors > 0) { + String message = String.format(" %d errors in file of length %d", + errors, len); + log.warn(message); + // the range either side of the first error to print + // this is a purely arbitrary number, to aid user debugging + final int overlap = 10; + for (int i = Math.max(0, first_error_byte - overlap); + i < Math.min(first_error_byte + overlap, len); + i++) { + byte actual = received[i]; + byte expected = original[i]; + String letter = toChar(actual); + String line = String.format("[%04d] %2x %s\n", i, actual, letter); + if (expected != actual) { + line = String.format("[%04d] %2x %s -expected %2x %s\n", + i, + actual, + letter, + expected, + toChar(expected)); + } + log.warn(line); + } + Assert.fail(message); + } + } + /** + * Convert a byte to a character for printing. If the + * byte value is < 32 -and hence unprintable- the byte is + * returned as a two digit hex value + * @param b byte + * @return the printable character string + */ + public static String toChar(byte b) { + if (b >= 0x20) { + return Character.toString((char) b); + } else { + return String.format("%02x", b); + } + } + + /** + * Convert a buffer to a string, character by character + * @param buffer input bytes + * @return a string conversion + */ + public static String toChar(byte[] buffer) { + StringBuilder builder = new StringBuilder(buffer.length); + for (byte b : buffer) { + builder.append(toChar(b)); + } + return builder.toString(); + } + + public static byte[] toAsciiByteArray(String s) { + char[] chars = s.toCharArray(); + int len = chars.length; + byte[] buffer = new byte[len]; + for (int i = 0; i < len; i++) { + buffer[i] = (byte) (chars[i] & 0xff); + } + return buffer; + } + + /** + * Create a dataset for use in the tests; all data is in the range + * base to (base+modulo-1) inclusive + * @param len length of data + * @param base base of the data + * @param modulo the modulo + * @return the newly generated dataset + */ + public static byte[] dataset(int len, int base, int modulo) { + byte[] dataset = new byte[len]; + for (int i = 0; i < len; i++) { + dataset[i] = (byte) (base + (i % modulo)); + } + return dataset; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
