Repository: hadoop Updated Branches: refs/heads/yarn-native-services 1c391dbeb -> 8f4b18957
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java new file mode 100644 index 0000000..7eaaefe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java @@ -0,0 +1,901 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.test; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.internal.AssumptionViolatedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Properties; +import java.util.UUID; + +/** + * Utilities used across test cases to make assertions about filesystems + * -assertions which fail with useful information. + * This is lifted from Hadoop common Test; that JAR isn't published, so + * we have to make do. + */ +public class ContractTestUtils extends Assert { + + private static final Logger LOG = + LoggerFactory.getLogger(ContractTestUtils.class); + + public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; + + // For scale testing, we can repeatedly write small chunk data to generate + // a large file. + public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size"; + public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128; + public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size"; + public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128; + + /** + * Assert that a property in the property set matches the expected value + * @param props property set + * @param key property name + * @param expected expected value. If null, the property must not be in the set + */ + public static void assertPropertyEquals(Properties props, + String key, + String expected) { + String val = props.getProperty(key); + if (expected == null) { + assertNull("Non null property " + key + " = " + val, val); + } else { + assertEquals("property " + key + " = " + val, + expected, + val); + } + } + + /** + * + * Write a file and read it in, validating the result. Optional flags control + * whether file overwrite operations should be enabled, and whether the + * file should be deleted afterwards. + * + * If there is a mismatch between what was written and what was expected, + * a small range of bytes either side of the first error are logged to aid + * diagnosing what problem occurred -whether it was a previous file + * or a corrupting of the current file. This assumes that two + * sequential runs to the same path use datasets with different character + * moduli. + * + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @param delete should the file be deleted afterwards? -with a verification + * that it worked. Deletion is not attempted if an assertion has failed + * earlier -it is not in a <code>finally{}</code> block. + * @throws IOException IO problems + */ + public static void writeAndRead(FileSystem fs, + Path path, + byte[] src, + int len, + int blocksize, + boolean overwrite, + boolean delete) throws IOException { + fs.mkdirs(path.getParent()); + + writeDataset(fs, path, src, len, blocksize, overwrite); + + byte[] dest = readDataset(fs, path, len); + + compareByteArrays(src, dest, len); + + if (delete) { + rejectRootOperation(path); + boolean deleted = fs.delete(path, false); + assertTrue("Deleted", deleted); + assertPathDoesNotExist(fs, "Cleanup failed", path); + } + } + + /** + * Write a file. + * Optional flags control + * whether file overwrite operations should be enabled + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @throws IOException IO problems + */ + public static void writeDataset(FileSystem fs, + Path path, + byte[] src, + int len, + int buffersize, + boolean overwrite) throws IOException { + assertTrue( + "Not enough data in source array to write " + len + " bytes", + src.length >= len); + FSDataOutputStream out = fs.create(path, + overwrite, + fs.getConf() + .getInt(IO_FILE_BUFFER_SIZE, + 4096), + (short) 1, + buffersize); + out.write(src, 0, len); + out.close(); + assertFileHasLength(fs, path, len); + } + + /** + * Read the file and convert to a byte dataset. + * This implements readfully internally, so that it will read + * in the file without ever having to seek() + * @param fs filesystem + * @param path path to read from + * @param len length of data to read + * @return the bytes + * @throws IOException IO problems + */ + public static byte[] readDataset(FileSystem fs, Path path, int len) + throws IOException { + FSDataInputStream in = fs.open(path); + byte[] dest = new byte[len]; + int offset =0; + int nread = 0; + try { + while (nread < len) { + int nbytes = in.read(dest, offset + nread, len - nread); + if (nbytes < 0) { + throw new EOFException("End of file reached before reading fully."); + } + nread += nbytes; + } + } finally { + in.close(); + } + return dest; + } + + /** + * Read a file, verify its length and contents match the expected array + * @param fs filesystem + * @param path path to file + * @param original original dataset + * @throws IOException IO Problems + */ + public static void verifyFileContents(FileSystem fs, + Path path, + byte[] original) throws IOException { + FileStatus stat = fs.getFileStatus(path); + String statText = stat.toString(); + assertTrue("not a file " + statText, stat.isFile()); + assertEquals("wrong length " + statText, original.length, stat.getLen()); + byte[] bytes = readDataset(fs, path, original.length); + compareByteArrays(original,bytes,original.length); + } + + /** + * Verify that the read at a specific offset in a stream + * matches that expected + * @param stm stream + * @param fileContents original file contents + * @param seekOff seek offset + * @param toRead number of bytes to read + * @throws IOException IO problems + */ + public static void verifyRead(FSDataInputStream stm, byte[] fileContents, + int seekOff, int toRead) throws IOException { + byte[] out = new byte[toRead]; + stm.seek(seekOff); + stm.readFully(out); + byte[] expected = Arrays.copyOfRange(fileContents, seekOff, + seekOff + toRead); + compareByteArrays(expected, out,toRead); + } + + /** + * Assert that tthe array original[0..len] and received[] are equal. + * A failure triggers the logging of the bytes near where the first + * difference surfaces. + * @param original source data + * @param received actual + * @param len length of bytes to compare + */ + public static void compareByteArrays(byte[] original, + byte[] received, + int len) { + assertEquals("Number of bytes read != number written", + len, received.length); + int errors = 0; + int first_error_byte = -1; + for (int i = 0; i < len; i++) { + if (original[i] != received[i]) { + if (errors == 0) { + first_error_byte = i; + } + errors++; + } + } + + if (errors > 0) { + String message = String.format(" %d errors in file of length %d", + errors, len); + LOG.warn(message); + // the range either side of the first error to print + // this is a purely arbitrary number, to aid user debugging + final int overlap = 10; + for (int i = Math.max(0, first_error_byte - overlap); + i < Math.min(first_error_byte + overlap, len); + i++) { + byte actual = received[i]; + byte expected = original[i]; + String letter = toChar(actual); + String line = String.format("[%04d] %2x %s\n", i, actual, letter); + if (expected != actual) { + line = String.format("[%04d] %2x %s -expected %2x %s\n", + i, + actual, + letter, + expected, + toChar(expected)); + } + LOG.warn(line); + } + fail(message); + } + } + + /** + * Convert a byte to a character for printing. If the + * byte value is < 32 -and hence unprintable- the byte is + * returned as a two digit hex value + * @param b byte + * @return the printable character string + */ + public static String toChar(byte b) { + if (b >= 0x20) { + return Character.toString((char) b); + } else { + return String.format("%02x", b); + } + } + + /** + * Convert a buffer to a string, character by character + * @param buffer input bytes + * @return a string conversion + */ + public static String toChar(byte[] buffer) { + StringBuilder builder = new StringBuilder(buffer.length); + for (byte b : buffer) { + builder.append(toChar(b)); + } + return builder.toString(); + } + + public static byte[] toAsciiByteArray(String s) { + char[] chars = s.toCharArray(); + int len = chars.length; + byte[] buffer = new byte[len]; + for (int i = 0; i < len; i++) { + buffer[i] = (byte) (chars[i] & 0xff); + } + return buffer; + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param cleanupPath path to delete as a string + */ + public static void cleanup(String action, + FileSystem fileSystem, + String cleanupPath) { + if (fileSystem == null) { + return; + } + Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(), + fileSystem.getWorkingDirectory()); + cleanup(action, fileSystem, path); + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + */ + public static void cleanup(String action, FileSystem fileSystem, Path path) { + noteAction(action); + try { + rm(fileSystem, path, true, false); + } catch (Exception e) { + LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e); + } + } + + /** + * Delete a directory. There's a safety check for operations against the + * root directory -these are intercepted and rejected with an IOException + * unless the allowRootDelete flag is true + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + * @param recursive flag to enable recursive delete + * @param allowRootDelete can the root directory be deleted? + * @throws IOException on any problem. + */ + public static boolean rm(FileSystem fileSystem, + Path path, + boolean recursive, + boolean allowRootDelete) throws + IOException { + if (fileSystem != null) { + rejectRootOperation(path, allowRootDelete); + if (fileSystem.exists(path)) { + return fileSystem.delete(path, recursive); + } + } + return false; + + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @param allowRootOperation can the root directory be manipulated? + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path, + boolean allowRootOperation) throws IOException { + if (path.isRoot() && !allowRootOperation) { + throw new IOException("Root directory operation rejected: " + path); + } + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path) throws IOException { + rejectRootOperation(path, false); + } + + + public static void noteAction(String action) { + if (LOG.isDebugEnabled()) { + LOG.debug("============== "+ action +" ============="); + } + } + + /** + * downgrade a failure to a message and a warning, then an + * exception for the Junit test runner to mark as failed + * @param message text message + * @param failure what failed + * @throws AssumptionViolatedException always + */ + public static void downgrade(String message, Throwable failure) { + LOG.warn("Downgrading test " + message, failure); + AssumptionViolatedException ave = + new AssumptionViolatedException(failure, null); + throw ave; + } + + /** + * report an overridden test as unsupported + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void unsupported(String message) { + skip(message); + } + + /** + * report a test has been skipped for some reason + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void skip(String message) { + LOG.info("Skipping: {}", message); + throw new AssumptionViolatedException(message); + } + + /** + * Fail with an exception that was received + * @param text text to use in the exception + * @param thrown a (possibly null) throwable to init the cause with + * @throws AssertionError with the text and throwable -always + */ + public static void fail(String text, Throwable thrown) { + AssertionError e = new AssertionError(text); + e.initCause(thrown); + throw e; + } + + /** + * Make an assertion about the length of a file + * @param fs filesystem + * @param path path of the file + * @param expected expected length + * @throws IOException on File IO problems + */ + public static void assertFileHasLength(FileSystem fs, Path path, + int expected) throws IOException { + FileStatus status = fs.getFileStatus(path); + assertEquals( + "Wrong file length of file " + path + " status: " + status, + expected, + status.getLen()); + } + + /** + * Assert that a path refers to a directory + * @param fs filesystem + * @param path path of the directory + * @throws IOException on File IO problems + */ + public static void assertIsDirectory(FileSystem fs, + Path path) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertIsDirectory(fileStatus); + } + + /** + * Assert that a path refers to a directory + * @param fileStatus stats to check + */ + public static void assertIsDirectory(FileStatus fileStatus) { + assertTrue("Should be a directory -but isn't: " + fileStatus, + fileStatus.isDirectory()); + } + + /** + * Write the text to a file, returning the converted byte array + * for use in validating the round trip + * @param fs filesystem + * @param path path of file + * @param text text to write + * @param overwrite should the operation overwrite any existing file? + * @return the read bytes + * @throws IOException on IO problems + */ + public static byte[] writeTextFile(FileSystem fs, + Path path, + String text, + boolean overwrite) throws IOException { + byte[] bytes = new byte[0]; + if (text != null) { + bytes = toAsciiByteArray(text); + } + createFile(fs, path, overwrite, bytes); + return bytes; + } + + /** + * Create a file + * @param fs filesystem + * @param path path to write + * @param overwrite overwrite flag + * @param data source dataset. Can be null + * @throws IOException on any problem + */ + public static void createFile(FileSystem fs, + Path path, + boolean overwrite, + byte[] data) throws IOException { + FSDataOutputStream stream = fs.create(path, overwrite); + if (data != null && data.length > 0) { + stream.write(data); + } + stream.close(); + } + + /** + * Touch a file + * @param fs filesystem + * @param path path + * @throws IOException IO problems + */ + public static void touch(FileSystem fs, + Path path) throws IOException { + createFile(fs, path, true, null); + } + + /** + * Delete a file/dir and assert that delete() returned true + * <i>and</i> that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive) throws IOException { + assertDeleted(fs, file, recursive, false); + } + + /** + * Delete a file/dir and assert that delete() returned true + * <i>and</i> that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @param allowRootOperations can the root dir be deleted? + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive, + boolean allowRootOperations) throws IOException { + rejectRootOperation(file, allowRootOperations); + assertPathExists(fs, "about to be deleted file", file); + boolean deleted = fs.delete(file, recursive); + String dir = ls(fs, file.getParent()); + assertTrue("Delete failed on " + file + ": " + dir, deleted); + assertPathDoesNotExist(fs, "Deleted file", file); + } + + /** + * Read in "length" bytes, convert to an ascii string + * @param fs filesystem + * @param path path to read + * @param length #of bytes to read. + * @return the bytes read and converted to a string + * @throws IOException IO problems + */ + public static String readBytesToString(FileSystem fs, + Path path, + int length) throws IOException { + FSDataInputStream in = fs.open(path); + try { + byte[] buf = new byte[length]; + in.readFully(0, buf); + return toChar(buf); + } finally { + in.close(); + } + } + + /** + * Take an array of filestats and convert to a string (prefixed w/ a [01] counter + * @param stats array of stats + * @param separator separator after every entry + * @return a stringified set + */ + public static String fileStatsToString(FileStatus[] stats, String separator) { + StringBuilder buf = new StringBuilder(stats.length * 128); + for (int i = 0; i < stats.length; i++) { + buf.append(String.format("[%02d] %s", i, stats[i])).append(separator); + } + return buf.toString(); + } + + /** + * List a directory + * @param fileSystem FS + * @param path path + * @return a directory listing or failure message + * @throws IOException + */ + public static String ls(FileSystem fileSystem, Path path) throws IOException { + if (path == null) { + //surfaces when someone calls getParent() on something at the top of the path + return "/"; + } + FileStatus[] stats; + String pathtext = "ls " + path; + try { + stats = fileSystem.listStatus(path); + } catch (FileNotFoundException e) { + return pathtext + " -file not found"; + } catch (IOException e) { + return pathtext + " -failed: " + e; + } + return dumpStats(pathtext, stats); + } + + public static String dumpStats(String pathname, FileStatus[] stats) { + return pathname + fileStatsToString(stats, "\n"); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param fileSystem filesystem to resolve path against + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + public static void assertIsFile(FileSystem fileSystem, Path filename) throws + IOException { + assertPathExists(fileSystem, "Expected file", filename); + FileStatus status = fileSystem.getFileStatus(filename); + assertIsFile(filename, status); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param filename name of the file + * @param status file status + */ + public static void assertIsFile(Path filename, FileStatus status) { + String fileInfo = filename + " " + status; + assertFalse("File claims to be a directory " + fileInfo, + status.isDirectory()); + assertFalse("File claims to be a symlink " + fileInfo, + status.isSymlink()); + } + + /** + * Create a dataset for use in the tests; all data is in the range + * base to (base+modulo-1) inclusive + * @param len length of data + * @param base base of the data + * @param modulo the modulo + * @return the newly generated dataset + */ + public static byte[] dataset(int len, int base, int modulo) { + byte[] dataset = new byte[len]; + for (int i = 0; i < len; i++) { + dataset[i] = (byte) (base + (i % modulo)); + } + return dataset; + } + + /** + * Assert that a path exists -but make no assertions as to the + * type of that entry + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws FileNotFoundException raised if the path is missing + * @throws IOException IO problems + */ + public static void assertPathExists(FileSystem fileSystem, String message, + Path path) throws IOException { + if (!fileSystem.exists(path)) { + //failure, report it + String listing = ls(fileSystem, path.getParent()); + throw new FileNotFoundException(message + ": not found " + path + + " in \"" + path.getParent() + "\" :\n" + listing); + } + } + + /** + * Assert that a path does not exist + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist(FileSystem fileSystem, + String message, + Path path) throws IOException { + try { + FileStatus status = fileSystem.getFileStatus(path); + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** + * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry + * @param fs filesystem + * @param dir directory to scan + * @param subdir full path to look for + * @throws IOException IO probles + */ + public static void assertListStatusFinds(FileSystem fs, + Path dir, + Path subdir) throws IOException { + FileStatus[] stats = fs.listStatus(dir); + boolean found = false; + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.toString()).append('\n'); + if (stat.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + /** + * Test for the host being an OSX machine + * @return true if the JVM thinks that is running on OSX + */ + public static boolean isOSX() { + return System.getProperty("os.name").contains("OS X"); + } + + /** + * compare content of file operations using a double byte array + * @param concat concatenated files + * @param bytes bytes + */ + public static void validateFileContent(byte[] concat, byte[][] bytes) { + int idx = 0; + boolean mismatch = false; + + for (byte[] bb : bytes) { + for (byte b : bb) { + if (b != concat[idx++]) { + mismatch = true; + break; + } + } + if (mismatch) + break; + } + assertFalse("File content of file is not as expected at offset " + idx, + mismatch); + } + + /** + * Receives test data from the given input file and checks the size of the + * data as well as the pattern inside the received data. + * + * @param fs FileSystem + * @param path Input file to be checked + * @param expectedSize the expected size of the data to be read from the + * input file in bytes + * @param bufferLen Pattern length + * @param modulus Pattern modulus + * @throws IOException + * thrown if an error occurs while reading the data + */ + public static void verifyReceivedData(FileSystem fs, Path path, + final long expectedSize, + final int bufferLen, + final int modulus) throws IOException { + final byte[] testBuffer = new byte[bufferLen]; + + long totalBytesRead = 0; + int nextExpectedNumber = 0; + final InputStream inputStream = fs.open(path); + try { + while (true) { + final int bytesRead = inputStream.read(testBuffer); + if (bytesRead < 0) { + break; + } + + totalBytesRead += bytesRead; + + for (int i = 0; i < bytesRead; ++i) { + if (testBuffer[i] != nextExpectedNumber) { + throw new IOException("Read number " + testBuffer[i] + + " but expected " + nextExpectedNumber); + } + + ++nextExpectedNumber; + + if (nextExpectedNumber == modulus) { + nextExpectedNumber = 0; + } + } + } + + if (totalBytesRead != expectedSize) { + throw new IOException("Expected to read " + expectedSize + + " bytes but only received " + totalBytesRead); + } + } finally { + inputStream.close(); + } + } + + /** + * Generates test data of the given size according to some specific pattern + * and writes it to the provided output file. + * + * @param fs FileSystem + * @param path Test file to be generated + * @param size The size of the test data to be generated in bytes + * @param bufferLen Pattern length + * @param modulus Pattern modulus + * @throws IOException + * thrown if an error occurs while writing the data + */ + public static long generateTestFile(FileSystem fs, Path path, + final long size, + final int bufferLen, + final int modulus) throws IOException { + final byte[] testBuffer = new byte[bufferLen]; + for (int i = 0; i < testBuffer.length; ++i) { + testBuffer[i] = (byte) (i % modulus); + } + + final OutputStream outputStream = fs.create(path, false); + long bytesWritten = 0; + try { + while (bytesWritten < size) { + final long diff = size - bytesWritten; + if (diff < testBuffer.length) { + outputStream.write(testBuffer, 0, (int) diff); + bytesWritten += diff; + } else { + outputStream.write(testBuffer); + bytesWritten += testBuffer.length; + } + } + + return bytesWritten; + } finally { + outputStream.close(); + } + } + + /** + * Creates and reads a file with the given size. The test file is generated + * according to a specific pattern so it can be easily verified even if it's + * a multi-GB one. + * During the read phase the incoming data stream is also checked against + * this pattern. + * + * @param fs FileSystem + * @param parent Test file parent dir path + * @throws IOException + * thrown if an I/O error occurs while writing or reading the test file + */ + public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize) + throws IOException { + int testBufferSize = fs.getConf() + .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE); + int modulus = fs.getConf() + .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE); + + final String objectName = UUID.randomUUID().toString(); + final Path objectPath = new Path(parent, objectName); + + // Write test file in a specific pattern + assertEquals(fileSize, + generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus)); + assertPathExists(fs, "not created successful", objectPath); + + // Now read the same file back and verify its content + try { + verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus); + } finally { + // Delete test file + fs.delete(objectPath, false); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java new file mode 100644 index 0000000..d739324 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FileTxnLog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.net.BindException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + + +/** + * This is a version of the HBase ZK cluster cut out to be standalone + */ +public class MiniZooKeeperCluster { + private static final Logger LOG = LoggerFactory.getLogger( + MiniZooKeeperCluster.class); + + private static final int TICK_TIME = 2000; + private static final int CONNECTION_TIMEOUT = 30000; + public static final int MAX_CLIENT_CONNECTIONS = 1000; + + private boolean started; + + /** The default port. If zero, we use a random port. */ + private int defaultClientPort = 0; + + private int clientPort; + + private List<NIOServerCnxnFactory> standaloneServerFactoryList; + private List<ZooKeeperServer> zooKeeperServers; + private List<Integer> clientPortList; + + private int activeZKServerIndex; + private int tickTime = 0; + + private Configuration configuration; + + public MiniZooKeeperCluster() { + this(new Configuration()); + } + + public MiniZooKeeperCluster(Configuration configuration) { + this.started = false; + this.configuration = configuration; + activeZKServerIndex = -1; + zooKeeperServers = new ArrayList<ZooKeeperServer>(); + clientPortList = new ArrayList<Integer>(); + standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>(); + } + + public void setDefaultClientPort(int clientPort) { + if (clientPort <= 0) { + throw new IllegalArgumentException("Invalid default ZK client port: " + + clientPort); + } + this.defaultClientPort = clientPort; + } + + /** + * Selects a ZK client port. Returns the default port if specified. + * Otherwise, returns a random port. The random port is selected from the + * range between 49152 to 65535. These ports cannot be registered with IANA + * and are intended for dynamic allocation (see http://bit.ly/dynports). + */ + private int selectClientPort() { + if (defaultClientPort > 0) { + return defaultClientPort; + } + return 0xc000 + new Random().nextInt(0x3f00); + } + + public void setTickTime(int tickTime) { + this.tickTime = tickTime; + } + + public int getBackupZooKeeperServerNum() { + return zooKeeperServers.size() - 1; + } + + public int getZooKeeperServerNum() { + return zooKeeperServers.size(); + } + + // / XXX: From o.a.zk.t.ClientBase + private static void setupTestEnv() { + // during the tests we run with 100K prealloc in the logs. + // on windows systems prealloc of 64M was seen to take ~15seconds + // resulting in test failure (client timeout on first session). + // set env and directly in order to handle static init/gc issues + System.setProperty("zookeeper.preAllocSize", "100"); + FileTxnLog.setPreallocSize(100 * 1024); + } + + public int startup(File baseDir) throws IOException, InterruptedException { + return startup(baseDir, 1); + } + + /** + * @param baseDir + * @param numZooKeeperServers + * @return ClientPort server bound to, -1 if there was a + * binding problem and we couldn't pick another port. + * @throws IOException + * @throws InterruptedException + */ + public int startup(File baseDir, int numZooKeeperServers) throws IOException, + InterruptedException { + if (numZooKeeperServers <= 0) + return -1; + + setupTestEnv(); + shutdown(); + + int tentativePort = selectClientPort(); + + // running all the ZK servers + for (int i = 0; i < numZooKeeperServers; i++) { + File dir = new File(baseDir, "zookeeper_" + i).getAbsoluteFile(); + recreateDir(dir); + int tickTimeToUse; + if (this.tickTime > 0) { + tickTimeToUse = this.tickTime; + } else { + tickTimeToUse = TICK_TIME; + } + ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse); + NIOServerCnxnFactory standaloneServerFactory; + while (true) { + try { + standaloneServerFactory = new NIOServerCnxnFactory(); + standaloneServerFactory.configure( + new InetSocketAddress(tentativePort), + MAX_CLIENT_CONNECTIONS + ); + } catch (BindException e) { + LOG.debug("Failed binding ZK Server to client port: " + + tentativePort, e); + // We're told to use some port but it's occupied, fail + if (defaultClientPort > 0) return -1; + // This port is already in use, try to use another. + tentativePort = selectClientPort(); + continue; + } + break; + } + + // Start up this ZK server + standaloneServerFactory.startup(server); + if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for startup of standalone server"); + } + + // We have selected this port as a client port. + clientPortList.add(tentativePort); + standaloneServerFactoryList.add(standaloneServerFactory); + zooKeeperServers.add(server); + tentativePort++; //for the next server + } + + // set the first one to be active ZK; Others are backups + activeZKServerIndex = 0; + started = true; + clientPort = clientPortList.get(activeZKServerIndex); + LOG.info("Started MiniZK Cluster and connect 1 ZK server " + + "on client port: " + clientPort); + return clientPort; + } + + private void recreateDir(File dir) throws IOException { + if (dir.exists()) { + if (!FileUtil.fullyDelete(dir)) { + throw new IOException("Could not delete zk base directory: " + dir); + } + } + try { + dir.mkdirs(); + } catch (SecurityException e) { + throw new IOException("creating dir: " + dir, e); + } + } + + /** + * @throws IOException + */ + public void shutdown() throws IOException { + if (!started) { + return; + } + + // shut down all the zk servers + for (int i = 0; i < standaloneServerFactoryList.size(); i++) { + NIOServerCnxnFactory standaloneServerFactory = + standaloneServerFactoryList.get(i); + int clientPort = clientPortList.get(i); + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + } + for (ZooKeeperServer zkServer : zooKeeperServers) { + //explicitly close ZKDatabase since ZookeeperServer does not close them + zkServer.getZKDatabase().close(); + } + + // clear everything + started = false; + activeZKServerIndex = 0; + standaloneServerFactoryList.clear(); + clientPortList.clear(); + zooKeeperServers.clear(); + + LOG.info("Shutdown MiniZK cluster with all ZK servers"); + } + + /**@return clientPort return clientPort if there is another ZK backup can run + * when killing the current active; return -1, if there is no backups. + * @throws IOException + * @throws InterruptedException + */ + public int killCurrentActiveZooKeeperServer() throws IOException, + InterruptedException { + if (!started || activeZKServerIndex < 0) { + return -1; + } + + // Shutdown the current active one + NIOServerCnxnFactory standaloneServerFactory = + standaloneServerFactoryList.get(activeZKServerIndex); + int clientPort = clientPortList.get(activeZKServerIndex); + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + + zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close(); + + // remove the current active zk server + standaloneServerFactoryList.remove(activeZKServerIndex); + clientPortList.remove(activeZKServerIndex); + zooKeeperServers.remove(activeZKServerIndex); + LOG.info("Kill the current active ZK servers in the cluster " + + "on client port: " + clientPort); + + if (standaloneServerFactoryList.size() == 0) { + // there is no backup servers; + return -1; + } + clientPort = clientPortList.get(activeZKServerIndex); + LOG.info("Activate a backup zk server in the cluster " + + "on client port: " + clientPort); + // return the next back zk server's port + return clientPort; + } + + /** + * Kill one back up ZK servers + * @throws IOException + * @throws InterruptedException + */ + public void killOneBackupZooKeeperServer() throws IOException, + InterruptedException { + if (!started || activeZKServerIndex < 0 || + standaloneServerFactoryList.size() <= 1) { + return; + } + + int backupZKServerIndex = activeZKServerIndex + 1; + // Shutdown the current active one + NIOServerCnxnFactory standaloneServerFactory = + standaloneServerFactoryList.get(backupZKServerIndex); + int clientPort = clientPortList.get(backupZKServerIndex); + + standaloneServerFactory.shutdown(); + if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) { + throw new IOException("Waiting for shutdown of standalone server"); + } + + zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close(); + + // remove this backup zk server + standaloneServerFactoryList.remove(backupZKServerIndex); + clientPortList.remove(backupZKServerIndex); + zooKeeperServers.remove(backupZKServerIndex); + LOG.info("Kill one backup ZK servers in the cluster " + + "on client port: " + clientPort); + } + + // XXX: From o.a.zk.t.ClientBase + private static boolean waitForServerDown(int port, long timeout) { + long start = System.currentTimeMillis(); + while (true) { + try { + Socket sock = new Socket("localhost", port); + try { + OutputStream outstream = sock.getOutputStream(); + outstream.write("stat".getBytes()); + outstream.flush(); + } finally { + sock.close(); + } + } catch (IOException e) { + return true; + } + + if (System.currentTimeMillis() > start + timeout) { + break; + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + // ignore + } + } + return false; + } + + // XXX: From o.a.zk.t.ClientBase + private static boolean waitForServerUp(int port, long timeout) { + long start = System.currentTimeMillis(); + while (true) { + try { + Socket sock = new Socket("localhost", port); + BufferedReader reader = null; + try { + OutputStream outstream = sock.getOutputStream(); + outstream.write("stat".getBytes()); + outstream.flush(); + + Reader isr = new InputStreamReader(sock.getInputStream()); + reader = new BufferedReader(isr); + String line = reader.readLine(); + if (line != null && line.startsWith("Zookeeper version:")) { + return true; + } + } finally { + sock.close(); + if (reader != null) { + reader.close(); + } + } + } catch (IOException e) { + // ignore as this is expected + LOG.info("server localhost:" + port + " not up " + e); + } + + if (System.currentTimeMillis() > start + timeout) { + break; + } + try { + Thread.sleep(250); + } catch (InterruptedException e) { + // ignore + } + } + return false; + } + + public int getClientPort() { + return clientPort; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java new file mode 100644 index 0000000..78ce3e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.tools; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; +import org.junit.Assert; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * Various utility methods + * Byte comparison methods are from + * <code>org.apache.hadoop.fs.contract.ContractTestUtils</code> + */ +public class TestUtility { + protected static final Logger log = + LoggerFactory.getLogger(TestUtility.class); + + public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException { + for (File file : dirObj.listFiles()) { + if (file.isDirectory()) { + addDir(file, zipFile, prefix + file.getName() + File.separator); + } else { + log.info("Adding to zip - " + prefix + file.getName()); + zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName())); + IOUtils.copy(new FileInputStream(file), zipFile); + zipFile.closeArchiveEntry(); + } + } + } + + public static void zipDir(String zipFile, String dir) throws IOException { + File dirObj = new File(dir); + ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile)); + log.info("Creating : {}", zipFile); + try { + addDir(dirObj, out, ""); + } finally { + out.close(); + } + } + + public static String createAppPackage( + TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException { + String zipFileName; + File pkgPath = folder.newFolder(subDir); + File zipFile = new File(pkgPath, pkgName).getAbsoluteFile(); + zipFileName = zipFile.getAbsolutePath(); + TestUtility.zipDir(zipFileName, srcPath); + log.info("Created temporary zip file at {}", zipFileName); + return zipFileName; + } + + + /** + * Assert that tthe array original[0..len] and received[] are equal. + * A failure triggers the logging of the bytes near where the first + * difference surfaces. + * @param original source data + * @param received actual + * @param len length of bytes to compare + */ + public static void compareByteArrays(byte[] original, + byte[] received, + int len) { + Assert.assertEquals("Number of bytes read != number written", + len, received.length); + int errors = 0; + int first_error_byte = -1; + for (int i = 0; i < len; i++) { + if (original[i] != received[i]) { + if (errors == 0) { + first_error_byte = i; + } + errors++; + } + } + + if (errors > 0) { + String message = String.format(" %d errors in file of length %d", + errors, len); + log.warn(message); + // the range either side of the first error to print + // this is a purely arbitrary number, to aid user debugging + final int overlap = 10; + for (int i = Math.max(0, first_error_byte - overlap); + i < Math.min(first_error_byte + overlap, len); + i++) { + byte actual = received[i]; + byte expected = original[i]; + String letter = toChar(actual); + String line = String.format("[%04d] %2x %s\n", i, actual, letter); + if (expected != actual) { + line = String.format("[%04d] %2x %s -expected %2x %s\n", + i, + actual, + letter, + expected, + toChar(expected)); + } + log.warn(line); + } + Assert.fail(message); + } + } + /** + * Convert a byte to a character for printing. If the + * byte value is < 32 -and hence unprintable- the byte is + * returned as a two digit hex value + * @param b byte + * @return the printable character string + */ + public static String toChar(byte b) { + if (b >= 0x20) { + return Character.toString((char) b); + } else { + return String.format("%02x", b); + } + } + + /** + * Convert a buffer to a string, character by character + * @param buffer input bytes + * @return a string conversion + */ + public static String toChar(byte[] buffer) { + StringBuilder builder = new StringBuilder(buffer.length); + for (byte b : buffer) { + builder.append(toChar(b)); + } + return builder.toString(); + } + + public static byte[] toAsciiByteArray(String s) { + char[] chars = s.toCharArray(); + int len = chars.length; + byte[] buffer = new byte[len]; + for (int i = 0; i < len; i++) { + buffer[i] = (byte) (chars[i] & 0xff); + } + return buffer; + } + + /** + * Create a dataset for use in the tests; all data is in the range + * base to (base+modulo-1) inclusive + * @param len length of data + * @param base base of the data + * @param modulo the modulo + * @return the newly generated dataset + */ + public static byte[] dataset(int len, int base, int modulo) { + byte[] dataset = new byte[len]; + for (int i = 0; i < len; i++) { + dataset[i] = (byte) (base + (i % modulo)); + } + return dataset; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt new file mode 100644 index 0000000..a1d7780 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt @@ -0,0 +1,16 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml new file mode 100644 index 0000000..cb8eab2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml @@ -0,0 +1,98 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <application> + <name>STORM</name> + <comment>Apache Hadoop Stream processing framework</comment> + <version>0.9.1.2.1</version> + <components> + + <component> + <name>NIMBUS</name> + <category>MASTER</category> + <commandScript> + <script>scripts/nimbus.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_REST_API</name> + <category>MASTER</category> + <commandScript> + <script>scripts/rest_api.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>SUPERVISOR</name> + <category>SLAVE</category> + <commandScript> + <script>scripts/supervisor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_UI_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/ui_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>DRPC_SERVER</name> + <category>MASTER</category> + <commandScript> + <script>scripts/drpc_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <configFiles> + <configFile> + <type>xml</type> + <fileName>storm-site.xml</fileName> + <dictionaryName>storm-site</dictionaryName> + </configFile> + </configFiles> + </application> +</metainfo> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt new file mode 100644 index 0000000..a1d7780 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt @@ -0,0 +1,16 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml new file mode 100644 index 0000000..f86e687 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml @@ -0,0 +1,17 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo></metainfo> http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f4b1895/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml new file mode 100644 index 0000000..fbe9299 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml @@ -0,0 +1,180 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <application> + <name>STORM</name> + <comment>Apache Hadoop Stream processing framework</comment> + <version>0.9.1.2.1</version> + <exportedConfigs>storm-site</exportedConfigs> + + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>app.jmx</name> + <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value> + </export> + <export> + <name>app.monitor</name> + <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> + </export> + <export> + <name>app.metrics</name> + <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value> + </export> + <export> + <name>ganglia.ui</name> + <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value> + </export> + <export> + <name>nimbus.url</name> + <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value> + </export> + </exports> + </exportGroup> + </exportGroups> + + <commandOrders> + <commandOrder> + <command>NIMBUS-START</command> + <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED + </requires> + </commandOrder> + <commandOrder> + <command>SUPERVISOR-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>DRPC_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_REST_API-START</command> + <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_UI_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + </commandOrders> + + <components> + + <component> + <name>NIMBUS</name> + <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports> + <commandScript> + <script>scripts/nimbus.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_REST_API</name> + <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-app.jmx</appExports> + <commandScript> + <script>scripts/rest_api.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>SUPERVISOR</name> + <category>SLAVE</category> + <autoStartOnFailure>true</autoStartOnFailure> + <componentExports> + <componentExport> + <name>log_viewer_port</name> + <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value> + </componentExport> + </componentExports> + <commandScript> + <script>scripts/supervisor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_UI_SERVER</name> + <category>MASTER</category> + <publishConfig>true</publishConfig> + <appExports>QuickLinks-app.monitor</appExports> + <autoStartOnFailure>true</autoStartOnFailure> + <commandScript> + <script>scripts/ui_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>DRPC_SERVER</name> + <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <commandScript> + <script>scripts/drpc_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>ANOTHER_COMPONENT</name> + <category>MASTER</category> + <commands> + <command> + <exec>start command</exec> + </command> + <command> + <exec>stop command</exec> + <name>STOP</name> + </command> + </commands> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>tarball</type> + <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <packages> + <package> + <type>tarball</type> + <name>test-tarball-name.tgz</name> + </package> + </packages> + </application> +</metainfo> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org