Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.exceptions; + +/** + * Exception raised when the J/O mapping fails. + */ +public class SwiftJsonMarshallingException extends SwiftException { + + public SwiftJsonMarshallingException(String message) { + super(message); + } + + public SwiftJsonMarshallingException(String message, Throwable cause) { + super(message, cause); + } +}
Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.exceptions; + +import org.apache.hadoop.fs.Path; + +/** + * Exception raised when an operation is meant to work on a directory, but + * the target path is not a directory + */ +public class SwiftNotDirectoryException extends SwiftException { + private final Path path; + + public SwiftNotDirectoryException(Path path) { + this(path, ""); + } + + public SwiftNotDirectoryException(Path path, + String message) { + super(path.toString() + message); + this.path = path; + } + + public Path getPath() { + return path; + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.exceptions; + +/** + * Used to relay exceptions upstream from the inner implementation + * to the public API, where it is downgraded to a log+failure. + * Making it visible internally aids testing + */ +public class SwiftOperationFailedException extends SwiftException { + + public SwiftOperationFailedException(String message) { + super(message); + } + + public SwiftOperationFailedException(String message, Throwable cause) { + super(message, cause); + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.exceptions; + +/** + * Exception raised when trying to create a file that already exists + * and the overwrite flag is set to false. + */ +public class SwiftPathExistsException extends SwiftException { + public SwiftPathExistsException(String message) { + super(message); + } + + public SwiftPathExistsException(String message, Throwable cause) { + super(message, cause); + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.exceptions; + +import org.apache.commons.httpclient.HttpMethod; + +import java.net.URI; + +/** + * Exception raised if a Swift endpoint returned a HTTP response indicating + * the caller is being throttled. + */ +public class SwiftThrottledRequestException extends + SwiftInvalidResponseException { + public SwiftThrottledRequestException(String message, + String operation, + URI uri, + HttpMethod method) { + super(message, operation, uri, method); + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.exceptions; + +/** + * Exception raised on an unsupported feature in the FS API -such as + * <code>append()</code> + */ +public class SwiftUnsupportedFeatureException extends SwiftException { + + public SwiftUnsupportedFeatureException(String message) { + super(message); + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.http; + +import org.apache.commons.httpclient.methods.EntityEnclosingMethod; + +/** + * Implementation for SwiftRestClient to make copy requests. + * COPY is a method that came with WebDAV (RFC2518), and is not something that + * can be handled by all proxies en-route to a filesystem. + */ +class CopyMethod extends EntityEnclosingMethod { + + public CopyMethod(String uri) { + super(uri); + } + + /** + * @return http method name + */ + @Override + public String getName() { + return "COPY"; + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.http; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + +/** + * Variant of Hadoop Netutils exception wrapping with URI awareness and + * available in branch-1 too. + */ +public class ExceptionDiags { + private static final Log LOG = LogFactory.getLog(ExceptionDiags.class); + + /** text to point users elsewhere: {@value} */ + private static final String FOR_MORE_DETAILS_SEE + = " For more details see: "; + /** text included in wrapped exceptions if the host is null: {@value} */ + public static final String UNKNOWN_HOST = "(unknown)"; + /** Base URL of the Hadoop Wiki: {@value} */ + public static final String HADOOP_WIKI = "http://wiki.apache.org/hadoop/"; + + /** + * Take an IOException and a URI, wrap it where possible with + * something that includes the URI + * + * @param dest target URI + * @param operation operation + * @param exception the caught exception. + * @return an exception to throw + */ + public static IOException wrapException(final String dest, + final String operation, + final IOException exception) { + String action = operation + " " + dest; + String xref = null; + + if (exception instanceof ConnectException) { + xref = "ConnectionRefused"; + } else if (exception instanceof UnknownHostException) { + xref = "UnknownHost"; + } else if (exception instanceof SocketTimeoutException) { + xref = "SocketTimeout"; + } else if (exception instanceof NoRouteToHostException) { + xref = "NoRouteToHost"; + } + String msg = action + + " failed on exception: " + + exception; + if (xref != null) { + msg = msg + ";" + see(xref); + } + return wrapWithMessage(exception, msg); + } + + private static String see(final String entry) { + return FOR_MORE_DETAILS_SEE + HADOOP_WIKI + entry; + } + + @SuppressWarnings("unchecked") + private static <T extends IOException> T wrapWithMessage( + T exception, String msg) { + Class<? extends Throwable> clazz = exception.getClass(); + try { + Constructor<? extends Throwable> ctor = + clazz.getConstructor(String.class); + Throwable t = ctor.newInstance(msg); + return (T) (t.initCause(exception)); + } catch (Throwable e) { + LOG.warn("Unable to wrap exception of type " + + clazz + ": it has no (String) constructor", e); + return exception; + } + } + +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.swift.http; + +/** + * Response tuple from GET operations; combines the input stream with the content length + */ +public class HttpBodyContent { + private final long contentLength; + private final HttpInputStreamWithRelease inputStream; + + /** + * build a body response + * @param inputStream input stream from the operatin + * @param contentLength length of content; may be -1 for "don't know" + */ + public HttpBodyContent(HttpInputStreamWithRelease inputStream, + long contentLength) { + this.contentLength = contentLength; + this.inputStream = inputStream; + } + + public long getContentLength() { + return contentLength; + } + + public HttpInputStreamWithRelease getInputStream() { + return inputStream; + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.http; + +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException; +import org.apache.hadoop.fs.swift.util.SwiftUtils; + +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; + +/** + * This replaces the input stream release class from JetS3t and AWS; + * # Failures in the constructor are relayed up instead of simply logged. + * # it is set up to be more robust at teardown + * # release logic is thread safe + * Note that the thread safety of the inner stream contains no thread + * safety guarantees -this stream is not to be read across streams. + * The thread safety logic here is to ensure that even if somebody ignores + * that rule, the release code does not get entered twice -and that + * any release in one thread is picked up by read operations in all others. + */ +public class HttpInputStreamWithRelease extends InputStream { + + private static final Log LOG = + LogFactory.getLog(HttpInputStreamWithRelease.class); + private final URI uri; + private HttpMethod method; + //flag to say the stream is released -volatile so that read operations + //pick it up even while unsynchronized. + private volatile boolean released; + //volatile flag to verify that data is consumed. + private volatile boolean dataConsumed; + private InputStream inStream; + /** + * In debug builds, this is filled in with the construction-time + * stack, which is then included in logs from the finalize(), method. + */ + private final Exception constructionStack; + + /** + * Why the stream is closed + */ + private String reasonClosed = "unopened"; + + public HttpInputStreamWithRelease(URI uri, HttpMethod method) throws + IOException { + this.uri = uri; + this.method = method; + constructionStack = LOG.isDebugEnabled() ? new Exception("stack") : null; + if (method == null) { + throw new IllegalArgumentException("Null 'method' parameter "); + } + try { + inStream = method.getResponseBodyAsStream(); + } catch (IOException e) { + inStream = new ByteArrayInputStream(new byte[]{}); + throw releaseAndRethrow("getResponseBodyAsStream() in constructor -" + e, e); + } + } + + @Override + public void close() throws IOException { + release("close()", null); + } + + /** + * Release logic + * @param reason reason for release (used in debug messages) + * @param ex exception that is a cause -null for non-exceptional releases + * @return true if the release took place here + * @throws IOException if the abort or close operations failed. + */ + private synchronized boolean release(String reason, Exception ex) throws + IOException { + if (!released) { + reasonClosed = reason; + try { + if (LOG.isDebugEnabled()) { + LOG.debug("Releasing connection to " + uri + ": " + reason, ex); + } + if (method != null) { + if (!dataConsumed) { + method.abort(); + } + method.releaseConnection(); + } + if (inStream != null) { + //this guard may seem un-needed, but a stack trace seen + //on the JetS3t predecessor implied that it + //is useful + inStream.close(); + } + return true; + } finally { + //if something went wrong here, we do not want the release() operation + //to try and do anything in advance. + released = true; + dataConsumed = true; + } + } else { + return false; + } + } + + /** + * Release the method, using the exception as a cause + * @param operation operation that failed + * @param ex the exception which triggered it. + * @return the exception to throw + */ + private IOException releaseAndRethrow(String operation, IOException ex) { + try { + release(operation, ex); + } catch (IOException ioe) { + LOG.debug("Exception during release: " + operation + " - " + ioe, ioe); + //make this the exception if there was none before + if (ex == null) { + ex = ioe; + } + } + return ex; + } + + /** + * Assume that the connection is not released: throws an exception if it is + * @throws SwiftConnectionClosedException + */ + private synchronized void assumeNotReleased() throws SwiftConnectionClosedException { + if (released || inStream == null) { + throw new SwiftConnectionClosedException(reasonClosed); + } + } + + @Override + public int available() throws IOException { + assumeNotReleased(); + try { + return inStream.available(); + } catch (IOException e) { + throw releaseAndRethrow("available() failed -" + e, e); + } + } + + @Override + public int read() throws IOException { + assumeNotReleased(); + int read = 0; + try { + read = inStream.read(); + } catch (EOFException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("EOF exception " + e, e); + } + read = -1; + } catch (IOException e) { + throw releaseAndRethrow("read()", e); + } + if (read < 0) { + dataConsumed = true; + release("read() -all data consumed", null); + } + return read; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + SwiftUtils.validateReadArgs(b, off, len); + //if the stream is already closed, then report an exception. + assumeNotReleased(); + //now read in a buffer, reacting differently to different operations + int read; + try { + read = inStream.read(b, off, len); + } catch (EOFException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("EOF exception " + e, e); + } + read = -1; + } catch (IOException e) { + throw releaseAndRethrow("read(b, off, " + len + ")", e); + } + if (read < 0) { + dataConsumed = true; + release("read() -all data consumed", null); + } + return read; + } + + /** + * Finalizer does release the stream, but also logs at WARN level + * including the URI at fault + */ + @Override + protected void finalize() { + try { + if (release("finalize()", constructionStack)) { + LOG.warn("input stream of " + uri + + " not closed properly -cleaned up in finalize()"); + } + } catch (Exception e) { + //swallow anything that failed here + LOG.warn("Exception while releasing " + uri + "in finalizer", + e); + } + } + + @Override + public String toString() { + return "HttpInputStreamWithRelease working with " + uri + +" released=" + released + +" dataConsumed=" + dataConsumed; + } +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.http; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException; + +import java.net.URI; +import java.util.Properties; + +import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*; + +/** + * This class implements the binding logic between Hadoop configurations + * and the swift rest client. + * <p/> + * The swift rest client takes a Properties instance containing + * the string values it uses to bind to a swift endpoint. + * <p/> + * This class extracts the values for a specific filesystem endpoint + * and then builds an appropriate Properties file. + */ +public final class RestClientBindings { + private static final Log LOG = LogFactory.getLog(RestClientBindings.class); + + public static final String E_INVALID_NAME = "Invalid swift hostname '%s':" + + " hostname must in form container.service"; + + /** + * Public for testing : build the full prefix for use in resolving + * configuration items + * + * @param service service to use + * @return the prefix string <i>without any trailing "."</i> + */ + public static String buildSwiftInstancePrefix(String service) { + return SWIFT_SERVICE_PREFIX + service; + } + + /** + * Raise an exception for an invalid service name + * + * @param hostname hostname that was being parsed + * @return an exception to throw + */ + private static SwiftConfigurationException invalidName(String hostname) { + return new SwiftConfigurationException( + String.format(E_INVALID_NAME, hostname)); + } + + /** + * Get the container name from the hostname -the single element before the + * first "." in the hostname + * + * @param hostname hostname to split + * @return the container + * @throws SwiftConfigurationException + */ + public static String extractContainerName(String hostname) throws + SwiftConfigurationException { + int i = hostname.indexOf("."); + if (i <= 0) { + throw invalidName(hostname); + } + return hostname.substring(0, i); + } + + public static String extractContainerName(URI uri) throws + SwiftConfigurationException { + return extractContainerName(uri.getHost()); + } + + /** + * Get the service name from a longer hostname string + * + * @param hostname hostname + * @return the separated out service name + * @throws SwiftConfigurationException if the hostname was invalid + */ + public static String extractServiceName(String hostname) throws + SwiftConfigurationException { + int i = hostname.indexOf("."); + if (i <= 0) { + throw invalidName(hostname); + } + String service = hostname.substring(i + 1); + if (service.isEmpty() || service.contains(".")) { + //empty service contains dots in -not currently supported + throw invalidName(hostname); + } + return service; + } + + public static String extractServiceName(URI uri) throws + SwiftConfigurationException { + return extractServiceName(uri.getHost()); + } + + /** + * Build a properties instance bound to the configuration file -using + * the filesystem URI as the source of the information. + * + * @param fsURI filesystem URI + * @param conf configuration + * @return a properties file with the instance-specific properties extracted + * and bound to the swift client properties. + * @throws SwiftConfigurationException if the configuration is invalid + */ + public static Properties bind(URI fsURI, Configuration conf) throws + SwiftConfigurationException { + String host = fsURI.getHost(); + if (host == null || host.isEmpty()) { + //expect shortnames -> conf names + throw invalidName(host); + } + + String container = extractContainerName(host); + String service = extractServiceName(host); + + //build filename schema + String prefix = buildSwiftInstancePrefix(service); + if (LOG.isDebugEnabled()) { + LOG.debug("Filesystem " + fsURI + + " is using configuration keys " + prefix); + } + Properties props = new Properties(); + props.setProperty(SWIFT_SERVICE_PROPERTY, service); + props.setProperty(SWIFT_CONTAINER_PROPERTY, container); + copy(conf, prefix + DOT_AUTH_URL, props, SWIFT_AUTH_PROPERTY, true); + copy(conf, prefix + DOT_USERNAME, props, SWIFT_USERNAME_PROPERTY, true); + copy(conf, prefix + DOT_APIKEY, props, SWIFT_APIKEY_PROPERTY, false); + copy(conf, prefix + DOT_PASSWORD, props, SWIFT_PASSWORD_PROPERTY, + props.contains(SWIFT_APIKEY_PROPERTY) ? true : false); + copy(conf, prefix + DOT_TENANT, props, SWIFT_TENANT_PROPERTY, false); + copy(conf, prefix + DOT_REGION, props, SWIFT_REGION_PROPERTY, false); + copy(conf, prefix + DOT_HTTP_PORT, props, SWIFT_HTTP_PORT_PROPERTY, false); + copy(conf, prefix + + DOT_HTTPS_PORT, props, SWIFT_HTTPS_PORT_PROPERTY, false); + + copyBool(conf, prefix + DOT_PUBLIC, props, SWIFT_PUBLIC_PROPERTY, false); + copyBool(conf, prefix + DOT_LOCATION_AWARE, props, + SWIFT_LOCATION_AWARE_PROPERTY, false); + + return props; + } + + /** + * Extract a boolean value from the configuration and copy it to the + * properties instance. + * @param conf source configuration + * @param confKey key in the configuration file + * @param props destination property set + * @param propsKey key in the property set + * @param defVal default value + */ + private static void copyBool(Configuration conf, + String confKey, + Properties props, + String propsKey, + boolean defVal) { + boolean b = conf.getBoolean(confKey, defVal); + props.setProperty(propsKey, Boolean.toString(b)); + } + + private static void set(Properties props, String key, String optVal) { + if (optVal != null) { + props.setProperty(key, optVal); + } + } + + /** + * Copy a (trimmed) property from the configuration file to the properties file. + * <p/> + * If marked as required and not found in the configuration, an + * exception is raised. + * If not required -and missing- then the property will not be set. + * In this case, if the property is already in the Properties instance, + * it will remain untouched. + * + * @param conf source configuration + * @param confKey key in the configuration file + * @param props destination property set + * @param propsKey key in the property set + * @param required is the property required + * @throws SwiftConfigurationException if the property is required but was + * not found in the configuration instance. + */ + public static void copy(Configuration conf, String confKey, Properties props, + String propsKey, + boolean required) throws SwiftConfigurationException { + //TODO: replace. version compatibility issue conf.getTrimmed fails with NoSuchMethodError + String val = conf.get(confKey); + if (val != null) { + val = val.trim(); + } + if (required && val == null) { + throw new SwiftConfigurationException( + "Missing mandatory configuration option: " + + + confKey); + } + set(props, propsKey, val); + } + + +} Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java?rev=1526848&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java (added) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java Fri Sep 27 11:12:42 2013 @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.http; + +import org.apache.hadoop.util.VersionInfo; + +/** + * Constants used in the Swift REST protocol, + * and in the properties used to configure the {@link SwiftRestClient}. + */ +public class SwiftProtocolConstants { + /** + * Swift-specific header for authentication: {@value} + */ + public static final String HEADER_AUTH_KEY = "X-Auth-Token"; + + /** + * Default port used by Swift for HTTP + */ + public static final int SWIFT_HTTP_PORT = 8080; + + /** + * Default port used by Swift Auth for HTTPS + */ + public static final int SWIFT_HTTPS_PORT = 443; + + /** HTTP standard {@value} header */ + public static final String HEADER_RANGE = "Range"; + + /** HTTP standard {@value} header */ + public static final String HEADER_DESTINATION = "Destination"; + + /** HTTP standard {@value} header */ + public static final String HEADER_LAST_MODIFIED = "Last-Modified"; + + /** HTTP standard {@value} header */ + public static final String HEADER_CONTENT_LENGTH = "Content-Length"; + + /** HTTP standard {@value} header */ + public static final String HEADER_CONTENT_RANGE = "Content-Range"; + + /** + * Patten for range headers + */ + public static final String SWIFT_RANGE_HEADER_FORMAT_PATTERN = "bytes=%d-%d"; + + /** + * section in the JSON catalog provided after auth listing the swift FS: + * {@value} + */ + public static final String SERVICE_CATALOG_SWIFT = "swift"; + /** + * section in the JSON catalog provided after auth listing the cloudfiles; + * this is an alternate catalog entry name + * {@value} + */ + public static final String SERVICE_CATALOG_CLOUD_FILES = "cloudFiles"; + /** + * section in the JSON catalog provided after auth listing the object store; + * this is an alternate catalog entry name + * {@value} + */ + public static final String SERVICE_CATALOG_OBJECT_STORE = "object-store"; + + /** + * entry in the swift catalog defining the prefix used to talk to objects + * {@value} + */ + public static final String SWIFT_OBJECT_AUTH_ENDPOINT = + "/object_endpoint/"; + /** + * Swift-specific header: object manifest used in the final upload + * of a multipart operation: {@value} + */ + public static final String X_OBJECT_MANIFEST = "X-Object-Manifest"; + /** + * Swift-specific header -#of objects in a container: {@value} + */ + public static final String X_CONTAINER_OBJECT_COUNT = + "X-Container-Object-Count"; + /** + * Swift-specific header: no. of bytes used in a container {@value} + */ + public static final String X_CONTAINER_BYTES_USED = "X-Container-Bytes-Used"; + + /** + * Header to set when requesting the latest version of a file: : {@value} + */ + public static final String X_NEWEST = "X-Newest"; + + /** + * throttled response sent by some endpoints. + */ + public static final int SC_THROTTLED_498 = 498; + /** + * W3C recommended status code for throttled operations + */ + public static final int SC_TOO_MANY_REQUESTS_429 = 429; + + public static final String FS_SWIFT = "fs.swift"; + + /** + * Prefix for all instance-specific values in the configuration: {@value} + */ + public static final String SWIFT_SERVICE_PREFIX = FS_SWIFT + ".service."; + + /** + * timeout for all connections: {@value} + */ + public static final String SWIFT_CONNECTION_TIMEOUT = + FS_SWIFT + ".connect.timeout"; + + /** + * timeout for all connections: {@value} + */ + public static final String SWIFT_SOCKET_TIMEOUT = + FS_SWIFT + ".socket.timeout"; + + /** + * the default socket timeout in millis {@value}. + * This controls how long the connection waits for responses from + * servers. + */ + public static final int DEFAULT_SOCKET_TIMEOUT = 60000; + + /** + * connection retry count for all connections: {@value} + */ + public static final String SWIFT_RETRY_COUNT = + FS_SWIFT + ".connect.retry.count"; + + /** + * delay in millis between bulk (delete, rename, copy operations: {@value} + */ + public static final String SWIFT_THROTTLE_DELAY = + FS_SWIFT + ".connect.throttle.delay"; + + /** + * the default throttle delay in millis {@value} + */ + public static final int DEFAULT_THROTTLE_DELAY = 0; + + /** + * blocksize for all filesystems: {@value} + */ + public static final String SWIFT_BLOCKSIZE = + FS_SWIFT + ".blocksize"; + + /** + * the default blocksize for filesystems in KB: {@value} + */ + public static final int DEFAULT_SWIFT_BLOCKSIZE = 32 * 1024; + + /** + * partition size for all filesystems in KB: {@value} + */ + public static final String SWIFT_PARTITION_SIZE = + FS_SWIFT + ".partsize"; + + /** + * The default partition size for uploads: {@value} + */ + public static final int DEFAULT_SWIFT_PARTITION_SIZE = 4608*1024; + + /** + * request size for reads in KB: {@value} + */ + public static final String SWIFT_REQUEST_SIZE = + FS_SWIFT + ".requestsize"; + + /** + * The default reqeuest size for reads: {@value} + */ + public static final int DEFAULT_SWIFT_REQUEST_SIZE = 64; + + + public static final String HEADER_USER_AGENT="User-Agent"; + + /** + * The user agent sent in requests. + */ + public static final String SWIFT_USER_AGENT= "Apache Hadoop Swift Client " + + VersionInfo.getBuildVersion(); + + /** + * Key for passing the service name as a property -not read from the + * configuration : {@value} + */ + public static final String DOT_SERVICE = ".SERVICE-NAME"; + + /** + * Key for passing the container name as a property -not read from the + * configuration : {@value} + */ + public static final String DOT_CONTAINER = ".CONTAINER-NAME"; + + public static final String DOT_AUTH_URL = ".auth.url"; + public static final String DOT_TENANT = ".tenant"; + public static final String DOT_USERNAME = ".username"; + public static final String DOT_PASSWORD = ".password"; + public static final String DOT_HTTP_PORT = ".http.port"; + public static final String DOT_HTTPS_PORT = ".https.port"; + public static final String DOT_REGION = ".region"; + public static final String DOT_PROXY_HOST = ".proxy.host"; + public static final String DOT_PROXY_PORT = ".proxy.port"; + public static final String DOT_LOCATION_AWARE = ".location-aware"; + public static final String DOT_APIKEY = ".apikey"; + public static final String DOT_USE_APIKEY = ".useApikey"; + + /** + * flag to say use public URL + */ + public static final String DOT_PUBLIC = ".public"; + + public static final String SWIFT_SERVICE_PROPERTY = FS_SWIFT + DOT_SERVICE; + public static final String SWIFT_CONTAINER_PROPERTY = FS_SWIFT + DOT_CONTAINER; + + public static final String SWIFT_AUTH_PROPERTY = FS_SWIFT + DOT_AUTH_URL; + public static final String SWIFT_TENANT_PROPERTY = FS_SWIFT + DOT_TENANT; + public static final String SWIFT_USERNAME_PROPERTY = FS_SWIFT + DOT_USERNAME; + public static final String SWIFT_PASSWORD_PROPERTY = FS_SWIFT + DOT_PASSWORD; + public static final String SWIFT_APIKEY_PROPERTY = FS_SWIFT + DOT_APIKEY; + public static final String SWIFT_HTTP_PORT_PROPERTY = FS_SWIFT + DOT_HTTP_PORT; + public static final String SWIFT_HTTPS_PORT_PROPERTY = FS_SWIFT + + DOT_HTTPS_PORT; + public static final String SWIFT_REGION_PROPERTY = FS_SWIFT + DOT_REGION; + public static final String SWIFT_PUBLIC_PROPERTY = FS_SWIFT + DOT_PUBLIC; + + public static final String SWIFT_USE_API_KEY_PROPERTY = FS_SWIFT + DOT_USE_APIKEY; + + public static final String SWIFT_LOCATION_AWARE_PROPERTY = FS_SWIFT + + DOT_LOCATION_AWARE; + + public static final String SWIFT_PROXY_HOST_PROPERTY = FS_SWIFT + DOT_PROXY_HOST; + public static final String SWIFT_PROXY_PORT_PROPERTY = FS_SWIFT + DOT_PROXY_PORT; + public static final String HTTP_ROUTE_DEFAULT_PROXY = + "http.route.default-proxy"; + /** + * Topology to return when a block location is requested + */ + public static final String TOPOLOGY_PATH = "/swift/unknown"; + /** + * Block location to return when a block location is requested + */ + public static final String BLOCK_LOCATION = "/default-rack/swift"; + /** + * Default number of attempts to retry a connect request: {@value} + */ + static final int DEFAULT_RETRY_COUNT = 3; + /** + * Default timeout in milliseconds for connection requests: {@value} + */ + static final int DEFAULT_CONNECT_TIMEOUT = 15000; +}