IGNITE-2206: Fixed a bug with file system factory deserialization.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/04827ca7 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/04827ca7 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/04827ca7 Branch: refs/heads/ignite-2206 Commit: 04827ca7f2f96d50e890b98603f1a9f41e1c853a Parents: 061c222 Author: vozerov-gridgain <voze...@gridgain.com> Authored: Wed Dec 23 17:17:40 2015 +0300 Committer: vozerov-gridgain <voze...@gridgain.com> Committed: Wed Dec 23 17:17:40 2015 +0300 ---------------------------------------------------------------------- .../internal/processors/igfs/IgfsPaths.java | 55 +++++++++++++------- .../hadoop/fs/v1/IgniteHadoopFileSystem.java | 35 ++++++++----- .../hadoop/fs/v2/IgniteHadoopFileSystem.java | 41 +++++++++------ .../hadoop/fs/HadoopLazyConcurrentMap.java | 4 -- 4 files changed, 81 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java index cd34655..0bb581b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java @@ -27,6 +27,8 @@ import java.io.ObjectOutput; import java.io.ObjectOutputStream; import java.util.ArrayList; import java.util.List; + +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.igfs.IgfsMode; import org.apache.ignite.igfs.IgfsPath; import org.apache.ignite.internal.util.typedef.T2; @@ -41,7 +43,7 @@ public class IgfsPaths implements Externalizable { private static final long serialVersionUID = 0L; /** */ - private Object payload; + private byte[] payloadBytes; /** Default IGFS mode. */ private IgfsMode dfltMode; @@ -62,11 +64,27 @@ public class IgfsPaths implements Externalizable { * @param payload Payload. * @param dfltMode Default IGFS mode. * @param pathModes Path modes. + * @throws IgniteCheckedException If failed. */ - public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable List<T2<IgfsPath, IgfsMode>> pathModes) { - this.payload = payload; + public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable List<T2<IgfsPath, IgfsMode>> pathModes) + throws IgniteCheckedException { this.dfltMode = dfltMode; this.pathModes = pathModes; + + if (payload == null) + payloadBytes = null; + else { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + try (ObjectOutput oo = new ObjectOutputStream(baos)) { + oo.writeObject(payload); + } + catch (IOException e) { + throw new IgniteCheckedException("Failed to serialize secondary file system factory: " + payload, e); + } + + payloadBytes = baos.toByteArray(); + } } /** @@ -85,20 +103,25 @@ public class IgfsPaths implements Externalizable { /** * @return Payload. + * + * @throws IgniteCheckedException If failed to deserialize the payload. */ - @Nullable public Object getPayload() { - return payload; + @Nullable public Object getPayload() throws IgniteCheckedException { + if (payloadBytes == null) + return null; + else { + try (ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(payloadBytes))) { + return oi.readObject(); + } + catch (IOException | ClassNotFoundException e) { + throw new IgniteCheckedException("Failed to deserialize secondary file system factory. ", e); + } + } } /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - try (ObjectOutput oo = new ObjectOutputStream(baos)) { - oo.writeObject(payload); - } - - U.writeByteArray(out, baos.toByteArray()); + U.writeByteArray(out, payloadBytes); U.writeEnum(out, dfltMode); @@ -120,13 +143,7 @@ public class IgfsPaths implements Externalizable { /** {@inheritDoc} */ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - byte[] factoryBytes = U.readByteArray(in); - - assert factoryBytes != null; - - try (ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes))) { - payload = oi.readObject(); - } + payloadBytes = U.readByteArray(in); dfltMode = IgfsMode.fromOrdinal(in.readByte()); http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java index 545c905..20cd9ca 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java @@ -17,19 +17,6 @@ package org.apache.ignite.hadoop.fs.v1; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; @@ -43,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Progressable; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory; import org.apache.ignite.igfs.IgfsBlockLocation; @@ -71,6 +59,20 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lifecycle.LifecycleAware; import org.jetbrains.annotations.Nullable; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_BATCH_SIZE; import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_DIR; import static org.apache.ignite.igfs.IgfsMode.PROXY; @@ -329,7 +331,12 @@ public class IgniteHadoopFileSystem extends FileSystem { } if (initSecondary) { - factory = (HadoopFileSystemFactory)paths.getPayload(); + try { + factory = (HadoopFileSystemFactory) paths.getPayload(); + } + catch (IgniteCheckedException e) { + throw new IOException("Failed to get secondary file system factory.", e); + } A.ensure(factory != null, "Secondary file system factory should not be null."); http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java index d665d4c..59d870c 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java @@ -17,22 +17,6 @@ package org.apache.ignite.hadoop.fs.v2; -import java.io.BufferedOutputStream; -import java.io.Closeable; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +35,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.igfs.IgfsBlockLocation; import org.apache.ignite.igfs.IgfsFile; @@ -76,6 +61,23 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lifecycle.LifecycleAware; import org.jetbrains.annotations.Nullable; +import java.io.BufferedOutputStream; +import java.io.Closeable; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_BATCH_SIZE; import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_DIR; import static org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.getFsHadoopUser; @@ -336,7 +338,12 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea } if (initSecondary) { - factory = (HadoopAbstractFileSystemFactory)paths.getPayload(); + try { + factory = (HadoopAbstractFileSystemFactory) paths.getPayload(); + } + catch (IgniteCheckedException e) { + throw new IOException("Failed to get secondary file system factory.", e); + } A.ensure(secondaryUri != null, "File system factory uri should not be null."); http://git-wip-us.apache.org/repos/asf/ignite/blob/04827ca7/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java index 2b20639..5808d01 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java @@ -112,10 +112,6 @@ public class HadoopLazyConcurrentMap<K, V extends Closeable> { if (closed) return; - // TODO: debug: - System.out.println("##### closed: " + System.identityHashCode(this)); - Thread.dumpStack(); - closed = true; Exception err = null;