Repository: mina-sshd Updated Branches: refs/heads/master ba91a674d -> c5b163f2b
[SSHD-790] Allow user to specify charset to be used when decoding SFTP client file/folder name(s) Project: http://git-wip-us.apache.org/repos/asf/mina-sshd/repo Commit: http://git-wip-us.apache.org/repos/asf/mina-sshd/commit/c5b163f2 Tree: http://git-wip-us.apache.org/repos/asf/mina-sshd/tree/c5b163f2 Diff: http://git-wip-us.apache.org/repos/asf/mina-sshd/diff/c5b163f2 Branch: refs/heads/master Commit: c5b163f2bf3b69c313018c18428cca32bebe5c2c Parents: ba91a67 Author: Goldstein Lyor <[email protected]> Authored: Wed Dec 27 13:12:20 2017 +0200 Committer: Goldstein Lyor <[email protected]> Committed: Wed Dec 27 13:51:31 2017 +0200 ---------------------------------------------------------------------- README.md | 51 ++++++++++++++++++++ .../subsystem/sftp/AbstractSftpClient.java | 16 ++++-- .../subsystem/sftp/DefaultSftpClient.java | 14 ++++++ .../sshd/client/subsystem/sftp/SftpClient.java | 25 ++++++++++ .../client/subsystem/sftp/SftpFileSystem.java | 11 +++++ .../subsystem/sftp/SftpFileSystemProvider.java | 19 ++++++-- .../common/util/buffer/ByteArrayBuffer.java | 3 ++ 7 files changed, 130 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index f5783e2..fb6560b 100644 --- a/README.md +++ b/README.md @@ -664,6 +664,57 @@ throughout the SFTP server subsystem code. The accessor is registered/overwritte ``` +### SFTP received names encoding + +By default, the SFTP client uses UTF-8 to decode any referenced file/folder name. However, some servers do not properly encode such names, and +thus the "decoded" names by the client become corrupted, or even worse - cause an exception upon decoding attempt. The `SftpClient` exposes +a `get/setNameDecodingCharset` method which enables the user to modify the charset - even while the SFTP session is in progress - e.g.: + +```java + + try (SftpClient client = ...obtain an instance...) { + client.setNameDecodingCharset(Charset.forName("ISO-8859-8")); + for (DirEntry entry : client.readDir(...some path...)) { + ...handle entry assuming ISO-8859-8 encoded names... + } + + client.setNameDecodingCharset(Charset.forName("ISO-8859-4")); + for (DirEntry entry : client.readDir(...some other path...)) { + ...handle entry assuming ISO-8859-4 encoded names... + } + } + +``` + +The initial charset can be pre-configured on the client/session by using the `sftp-name-decoding-charset` property - if none specified then +UTF-8 is used. **Note:** the value can be a charset name or a `java.nio.charset.Charset` instance - e.g.: + +```java + + SshClient client = ... setup/obtain an instance... + // default for ALL SFTP clients obtained through this client + PropertyResolverUtils.updateProperty(client, SftpClient.NAME_DECODING_CHARSET, "ISO-8859-8"); + + try (ClientSession session = client.connect(...)) { + // default for ALL SFTP clients obtained through the session - overrides client setting + PropertyResolverUtils.updateProperty(session, SftpClient.NAME_DECODING_CHARSET, "ISO-8859-4"); + session.authenticate(...); + + try (SftpClient sftp = session.createSftpClient()) { + for (DirEntry entry : sftp.readDir(...some path...)) { + ...handle entry assuming ISO-8859-4 (inherited from the session) encoded names... + } + + // override the inherited default from the session + sftp.setNameDecodingCharset(Charset.forName("ISO-8859-1")); + + for (DirEntry entry : sftp.readDir(...some other path...)) { + ...handle entry assuming ISO-8859-1 encoded names... + } + } + } + +``` ### Supported SFTP extensions http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/AbstractSftpClient.java ---------------------------------------------------------------------- diff --git a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/AbstractSftpClient.java b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/AbstractSftpClient.java index fb1aed8..fd2875c 100644 --- a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/AbstractSftpClient.java +++ b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/AbstractSftpClient.java @@ -19,6 +19,7 @@ package org.apache.sshd.client.subsystem.sftp; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.file.attribute.FileTime; import java.util.ArrayList; import java.util.Collection; @@ -105,6 +106,11 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme return parsed; } + protected String getReferencedName(Buffer buf) { + Charset cs = getNameDecodingCharset(); + return buf.getString(cs); + } + /** * Sends the specified command, waits for the response and then invokes {@link #checkResponseStatus(int, Buffer)} * @param cmd The command to send @@ -285,11 +291,11 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme if (len != 1) { throw new SshException("SFTP error: received " + len + " names instead of 1"); } - String name = buffer.getString(); + String name = getReferencedName(buffer); String longName = null; int version = getVersion(); if (version == SftpConstants.SFTP_V3) { - longName = buffer.getString(); + longName = getReferencedName(buffer); } Attributes attrs = readAttributes(buffer); @@ -417,7 +423,7 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme } if ((flags & SftpConstants.SSH_FILEXFER_ATTR_UNTRANSLATED_NAME) != 0) { @SuppressWarnings("unused") - String untranslated = buffer.getString(); // TODO: handle untranslated-name + String untranslated = getReferencedName(buffer); // TODO: handle untranslated-name } } } else { @@ -893,8 +899,8 @@ public abstract class AbstractSftpClient extends AbstractSubsystemClient impleme List<DirEntry> entries = new ArrayList<>(len); for (int i = 0; i < len; i++) { - String name = buffer.getString(); - String longName = (version == SftpConstants.SFTP_V3) ? buffer.getString() : null; + String name = getReferencedName(buffer); + String longName = (version == SftpConstants.SFTP_V3) ? getReferencedName(buffer) : null; Attributes attrs = readAttributes(buffer); if (log.isTraceEnabled()) { log.trace("checkDirResponse({})[id={}][{}] ({})[{}]: {}", http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/DefaultSftpClient.java ---------------------------------------------------------------------- diff --git a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/DefaultSftpClient.java b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/DefaultSftpClient.java index b72502c..0788b67 100644 --- a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/DefaultSftpClient.java +++ b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/DefaultSftpClient.java @@ -26,6 +26,7 @@ import java.io.InterruptedIOException; import java.io.OutputStream; import java.io.StreamCorruptedException; import java.net.SocketTimeoutException; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -43,6 +44,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.sshd.client.channel.ChannelSubsystem; import org.apache.sshd.client.channel.ClientChannel; import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.common.PropertyResolverUtils; import org.apache.sshd.common.SshException; import org.apache.sshd.common.subsystem.sftp.SftpConstants; import org.apache.sshd.common.subsystem.sftp.extensions.ParserUtils; @@ -67,8 +69,10 @@ public class DefaultSftpClient extends AbstractSftpClient { private final AtomicBoolean closing = new AtomicBoolean(false); private final NavigableMap<String, byte[]> extensions = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); private final NavigableMap<String, byte[]> exposedExtensions = Collections.unmodifiableNavigableMap(extensions); + private Charset nameDecodingCharset = DEFAULT_NAME_DECODING_CHARSET; public DefaultSftpClient(ClientSession clientSession) throws IOException { + this.nameDecodingCharset = PropertyResolverUtils.getCharset(clientSession, NAME_DECODING_CHARSET, DEFAULT_NAME_DECODING_CHARSET); this.clientSession = Objects.requireNonNull(clientSession, "No client session"); this.channel = clientSession.createSubsystemChannel(SftpConstants.SFTP_SUBSYSTEM_NAME); this.channel.setOut(new OutputStream() { @@ -130,6 +134,16 @@ public class DefaultSftpClient extends AbstractSftpClient { } @Override + public Charset getNameDecodingCharset() { + return nameDecodingCharset; + } + + @Override + public void setNameDecodingCharset(Charset nameDecodingCharset) { + this.nameDecodingCharset = Objects.requireNonNull(nameDecodingCharset, "No charset provided"); + } + + @Override public boolean isClosing() { return closing.get(); } http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpClient.java ---------------------------------------------------------------------- diff --git a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpClient.java b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpClient.java index 410fa22..c835e35 100644 --- a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpClient.java +++ b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpClient.java @@ -23,6 +23,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.channels.Channel; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.AclEntry; @@ -53,6 +55,21 @@ import org.apache.sshd.common.util.buffer.BufferUtils; * @author <a href="http://mina.apache.org";>Apache MINA Project</a> */ public interface SftpClient extends SubsystemClient { + /** + * Used to indicate the {@link Charset} (or its name) for decoding + * referenced files/folders names - extracted from the client session + * when 1st initialized. + * @see #DEFAULT_NAME_DECODING_CHARSET + * @see #getNameDecodingCharset() + * @see #setNameDecodingCharset(Charset) + */ + String NAME_DECODING_CHARSET = "sftp-name-decoding-charset"; + + /** + * Default value of {@value #NAME_DECODING_CHARSET} + */ + Charset DEFAULT_NAME_DECODING_CHARSET = StandardCharsets.UTF_8; + enum OpenMode { Read, Write, @@ -540,6 +557,14 @@ public interface SftpClient extends SubsystemClient { } /** + * @return The (never {@code null}) {@link Charset} used to decode referenced files/folders names + * @see #NAME_DECODING_CHARSET + */ + Charset getNameDecodingCharset(); + + void setNameDecodingCharset(Charset cs); + + /** * @return An (unmodifiable) {@link NavigableMap} of the reported server extensions. * where key=extension name (case <U>insensitive</U>) */ http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystem.java ---------------------------------------------------------------------- diff --git a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystem.java b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystem.java index cd41691..f028a5b 100644 --- a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystem.java +++ b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystem.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.StreamCorruptedException; +import java.nio.charset.Charset; import java.nio.file.FileStore; import java.nio.file.FileSystemException; import java.nio.file.attribute.GroupPrincipal; @@ -241,6 +242,16 @@ public class SftpFileSystem extends BaseFileSystem<SftpPath> implements ClientSe } @Override + public Charset getNameDecodingCharset() { + return delegate.getNameDecodingCharset(); + } + + @Override + public void setNameDecodingCharset(Charset cs) { + delegate.setNameDecodingCharset(cs); + } + + @Override public boolean isClosing() { return false; } http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystemProvider.java ---------------------------------------------------------------------- diff --git a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystemProvider.java b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystemProvider.java index f8af4f5..5b2f49d 100644 --- a/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystemProvider.java +++ b/sshd-core/src/main/java/org/apache/sshd/client/subsystem/sftp/SftpFileSystemProvider.java @@ -25,6 +25,7 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; import java.nio.channels.FileChannel; +import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; import java.nio.file.AccessMode; import java.nio.file.CopyOption; @@ -59,9 +60,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.Objects; import java.util.Set; import java.util.TreeMap; @@ -100,6 +101,9 @@ public class SftpFileSystemProvider extends FileSystemProvider { public static final long DEFAULT_CONNECT_TIME = SftpClient.DEFAULT_WAIT_TIMEOUT; public static final String AUTH_TIME_PROP_NAME = "sftp-fs-auth-time"; public static final long DEFAULT_AUTH_TIME = SftpClient.DEFAULT_WAIT_TIMEOUT; + public static final String NAME_DECORDER_CHARSET_PROP_NAME = "sftp-fs-name-decoder-charset"; + public static final Charset DEFAULT_NAME_DECODER_CHARSET = SftpClient.DEFAULT_NAME_DECODING_CHARSET; + /** * <P> * URI parameter that can be used to specify a special version selection. Options are: @@ -125,7 +129,7 @@ public class SftpFileSystemProvider extends FileSystemProvider { private final SshClient client; private final SftpVersionSelector selector; - private final Map<String, SftpFileSystem> fileSystems = new HashMap<>(); + private final NavigableMap<String, SftpFileSystem> fileSystems = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); public SftpFileSystemProvider() { this((SshClient) null); @@ -183,6 +187,10 @@ public class SftpFileSystemProvider extends FileSystemProvider { Map<String, Object> params = resolveFileSystemParameters(env, parseURIParameters(uri)); PropertyResolver resolver = PropertyResolverUtils.toPropertyResolver(params); SftpVersionSelector selector = resolveSftpVersionSelector(uri, getSftpVersionSelector(), resolver); + Charset decodingCharset = + PropertyResolverUtils.getCharset(resolver, NAME_DECORDER_CHARSET_PROP_NAME, DEFAULT_NAME_DECODER_CHARSET); + long maxConnectTime = resolver.getLongProperty(CONNECT_TIME_PROP_NAME, DEFAULT_CONNECT_TIME); + long maxAuthTime = resolver.getLongProperty(AUTH_TIME_PROP_NAME, DEFAULT_AUTH_TIME); SftpFileSystem fileSystem; synchronized (fileSystems) { @@ -194,7 +202,7 @@ public class SftpFileSystemProvider extends FileSystemProvider { ClientSession session = null; try { session = client.connect(username, host, port) - .verify(resolver.getLongProperty(CONNECT_TIME_PROP_NAME, DEFAULT_CONNECT_TIME)) + .verify(maxConnectTime) .getSession(); if (GenericUtils.size(params) > 0) { // Cannot use forEach because the session is not effectively final @@ -207,9 +215,12 @@ public class SftpFileSystemProvider extends FileSystemProvider { PropertyResolverUtils.updateProperty(session, key, value); } + + PropertyResolverUtils.updateProperty(session, SftpClient.NAME_DECODING_CHARSET, decodingCharset); } + session.addPasswordIdentity(password); - session.auth().verify(resolver.getLongProperty(AUTH_TIME_PROP_NAME, DEFAULT_AUTH_TIME)); + session.auth().verify(maxAuthTime); fileSystem = new SftpFileSystem(this, id, session, selector); fileSystems.put(id, fileSystem); http://git-wip-us.apache.org/repos/asf/mina-sshd/blob/c5b163f2/sshd-core/src/main/java/org/apache/sshd/common/util/buffer/ByteArrayBuffer.java ---------------------------------------------------------------------- diff --git a/sshd-core/src/main/java/org/apache/sshd/common/util/buffer/ByteArrayBuffer.java b/sshd-core/src/main/java/org/apache/sshd/common/util/buffer/ByteArrayBuffer.java index 91e6498..6655ec1 100644 --- a/sshd-core/src/main/java/org/apache/sshd/common/util/buffer/ByteArrayBuffer.java +++ b/sshd-core/src/main/java/org/apache/sshd/common/util/buffer/ByteArrayBuffer.java @@ -22,6 +22,7 @@ package org.apache.sshd.common.util.buffer; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.Arrays; +import java.util.Objects; import java.util.function.IntUnaryOperator; import org.apache.sshd.common.util.GenericUtils; @@ -175,6 +176,8 @@ public class ByteArrayBuffer extends Buffer { throw new BufferException("Bad item length: " + len); } ensureAvailable(len); + + Objects.requireNonNull(charset, "No charset specified"); String s = new String(data, rpos, len, charset); rpos += len; return s;
