IGNITE-2811: IGFS: Optimized properties handling.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e886ad0a Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e886ad0a Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e886ad0a Branch: refs/heads/ignite-testing-discovery Commit: e886ad0aa800cddb3308fa5f8400902e5879ee3c Parents: 218132d Author: vozerov-gridgain <[email protected]> Authored: Tue Mar 22 10:28:13 2016 +0300 Committer: vozerov-gridgain <[email protected]> Committed: Tue Mar 29 15:14:07 2016 +0300 ---------------------------------------------------------------------- .../internal/processors/igfs/IgfsEntryInfo.java | 8 +- .../ignite/internal/processors/igfs/IgfsEx.java | 12 -- .../internal/processors/igfs/IgfsImpl.java | 2 +- .../processors/igfs/IgfsOutputStreamImpl.java | 2 +- .../internal/processors/igfs/IgfsUtils.java | 113 +++++++++++++++++++ .../meta/IgfsMetaDirectoryCreateProcessor.java | 8 +- .../igfs/meta/IgfsMetaFileCreateProcessor.java | 8 +- .../meta/IgfsMetaUpdatePropertiesProcessor.java | 5 +- .../processors/igfs/IgfsAbstractSelfTest.java | 20 ++-- .../igfs/IgfsDualAbstractSelfTest.java | 13 ++- .../processors/igfs/IgfsStreamsSelfTest.java | 2 +- .../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 8 +- .../hadoop/fs/v1/IgniteHadoopFileSystem.java | 19 ++-- .../hadoop/fs/v2/IgniteHadoopFileSystem.java | 21 ++-- .../hadoop/igfs/HadoopIgfsProperties.java | 11 +- ...oopFileSystemUniversalFileSystemAdapter.java | 8 +- .../processors/hadoop/HadoopMapReduceTest.java | 5 +- 17 files changed, 187 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java index d31ef72..45cf828 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java @@ -314,7 +314,9 @@ public abstract class IgfsEntryInfo implements Externalizable { */ protected void writeBinary(BinaryRawWriter out) { BinaryUtils.writeIgniteUuid(out, id); - out.writeMap(props); + + IgfsUtils.writeProperties(out, props); + out.writeLong(accessTime); out.writeLong(modificationTime); out.writeObject(path); @@ -327,7 +329,9 @@ public abstract class IgfsEntryInfo implements Externalizable { */ protected void readBinary(BinaryRawReader in) { id = BinaryUtils.readIgniteUuid(in); - props = in.readMap(); + + props = IgfsUtils.readProperties(in); + accessTime = in.readLong(); modificationTime = in.readLong(); path = in.readObject(); http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java index cf268e0..fb67e20 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java @@ -31,18 +31,6 @@ import org.jetbrains.annotations.Nullable; * Internal API extension for {@link org.apache.ignite.IgniteFileSystem}. */ public interface IgfsEx extends IgniteFileSystem { - /** File property: user name. */ - public static final String PROP_USER_NAME = "usrName"; - - /** File property: group name. */ - public static final String PROP_GROUP_NAME = "grpName"; - - /** File property: permission. */ - public static final String PROP_PERMISSION = "permission"; - - /** File property: prefer writes to local node. */ - public static final String PROP_PREFER_LOCAL_WRITES = "locWrite"; - /** * Stops IGFS cleaning all used resources. * http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index 398428a..e3a82a5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -121,7 +121,7 @@ public final class IgfsImpl implements IgfsEx { private static final String PERMISSION_DFLT_VAL = "0777"; /** Default directory metadata. */ - static final Map<String, String> DFLT_DIR_META = F.asMap(PROP_PERMISSION, PERMISSION_DFLT_VAL); + static final Map<String, String> DFLT_DIR_META = F.asMap(IgfsUtils.PROP_PERMISSION, PERMISSION_DFLT_VAL); /** Handshake message. */ private final IgfsPaths secondaryPaths; http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java index ef2826b..f7c85e8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java @@ -412,7 +412,7 @@ class IgfsOutputStreamImpl extends IgfsOutputStreamAdapter { if (!igfsCtx.configuration().isFragmentizerEnabled()) return null; - if (!Boolean.parseBoolean(fileInfo.properties().get(IgfsEx.PROP_PREFER_LOCAL_WRITES))) + if (!Boolean.parseBoolean(fileInfo.properties().get(IgfsUtils.PROP_PREFER_LOCAL_WRITES))) return null; int blockSize = fileInfo.blockSize(); http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java index 94e1cef..9b813b8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsUtils.java @@ -46,6 +46,7 @@ import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.Nullable; import java.lang.reflect.Constructor; +import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; @@ -67,6 +68,33 @@ public class IgfsUtils { /** Constant trash concurrency level. */ public static final int TRASH_CONCURRENCY = 64; + /** File property: user name. */ + public static final String PROP_USER_NAME = "usrName"; + + /** File property: group name. */ + public static final String PROP_GROUP_NAME = "grpName"; + + /** File property: permission. */ + public static final String PROP_PERMISSION = "permission"; + + /** File property: prefer writes to local node. */ + public static final String PROP_PREFER_LOCAL_WRITES = "locWrite"; + + /** Generic property index. */ + private static final byte PROP_IDX = 0; + + /** User name property index. */ + private static final byte PROP_USER_NAME_IDX = 1; + + /** Group name property index. */ + private static final byte PROP_GROUP_NAME_IDX = 2; + + /** Permission property index. */ + private static final byte PROP_PERMISSION_IDX = 3; + + /** Prefer local writes property index. */ + private static final byte PROP_PREFER_LOCAL_WRITES_IDX = 4; + /** Trash directory IDs. */ private static final IgniteUuid[] TRASH_IDS; @@ -427,4 +455,89 @@ public class IgfsUtils { else return null; } + + /** + * Write entry properties. Rely on reference equality for well-known properties. + * + * @param out Writer. + * @param props Properties. + */ + @SuppressWarnings("StringEquality") + public static void writeProperties(BinaryRawWriter out, @Nullable Map<String, String> props) { + if (props != null) { + out.writeInt(props.size()); + + for (Map.Entry<String, String> entry : props.entrySet()) { + String key = entry.getKey(); + + if (key == PROP_PERMISSION) + out.writeByte(PROP_PERMISSION_IDX); + else if (key == PROP_PREFER_LOCAL_WRITES) + out.writeByte(PROP_PREFER_LOCAL_WRITES_IDX); + else if (key == PROP_USER_NAME) + out.writeByte(PROP_USER_NAME_IDX); + else if (key == PROP_GROUP_NAME) + out.writeByte(PROP_GROUP_NAME_IDX); + else { + out.writeByte(PROP_IDX); + out.writeString(key); + } + + out.writeString(entry.getValue()); + } + } + else + out.writeInt(-1); + } + + /** + * Read entry properties. + * + * @param in Reader. + * @return Properties. + */ + @Nullable public static Map<String, String> readProperties(BinaryRawReader in) { + int size = in.readInt(); + + if (size >= 0) { + Map<String, String> props = new HashMap<>(size); + + for (int i = 0; i < size; i++) { + byte idx = in.readByte(); + + String key; + + switch (idx) { + case PROP_PERMISSION_IDX: + key = PROP_PERMISSION; + + break; + + case PROP_PREFER_LOCAL_WRITES_IDX: + key = PROP_PREFER_LOCAL_WRITES; + + break; + + case PROP_USER_NAME_IDX: + key = PROP_USER_NAME; + + break; + + case PROP_GROUP_NAME_IDX: + key = PROP_GROUP_NAME; + + break; + + default: + key = in.readString(); + } + + props.put(key, in.readString()); + } + + return props; + } + else + return null; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaDirectoryCreateProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaDirectoryCreateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaDirectoryCreateProcessor.java index 907019c..5f4fe73 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaDirectoryCreateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaDirectoryCreateProcessor.java @@ -144,7 +144,9 @@ public class IgfsMetaDirectoryCreateProcessor implements EntryProcessor<IgniteUu BinaryRawWriter out = writer.rawWriter(); out.writeLong(createTime); - out.writeMap(props); + + IgfsUtils.writeProperties(out, props); + out.writeString(childName); if (childName != null) @@ -156,7 +158,9 @@ public class IgfsMetaDirectoryCreateProcessor implements EntryProcessor<IgniteUu BinaryRawReader in = reader.rawReader(); createTime = in.readLong(); - props = in.readMap(); + + props = IgfsUtils.readProperties(in); + childName = in.readString(); if (childName != null) http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaFileCreateProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaFileCreateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaFileCreateProcessor.java index 41745f1..9fd16aa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaFileCreateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaFileCreateProcessor.java @@ -138,7 +138,9 @@ public class IgfsMetaFileCreateProcessor implements EntryProcessor<IgniteUuid, I BinaryRawWriter out = writer.rawWriter(); out.writeLong(createTime); - out.writeMap(props); + + IgfsUtils.writeProperties(out, props); + out.writeInt(blockSize); BinaryUtils.writeIgniteUuid(out, affKey); BinaryUtils.writeIgniteUuid(out, lockId); @@ -150,7 +152,9 @@ public class IgfsMetaFileCreateProcessor implements EntryProcessor<IgniteUuid, I BinaryRawReader in = reader.rawReader(); createTime = in.readLong(); - props = in.readMap(); + + props = IgfsUtils.readProperties(in); + blockSize = in.readInt(); affKey = BinaryUtils.readIgniteUuid(in); lockId = BinaryUtils.readIgniteUuid(in); http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaUpdatePropertiesProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaUpdatePropertiesProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaUpdatePropertiesProcessor.java index fb4466b..e0d5b8b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaUpdatePropertiesProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/meta/IgfsMetaUpdatePropertiesProcessor.java @@ -24,6 +24,7 @@ import org.apache.ignite.binary.BinaryReader; import org.apache.ignite.binary.BinaryWriter; import org.apache.ignite.binary.Binarylizable; import org.apache.ignite.internal.processors.igfs.IgfsEntryInfo; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.util.GridLeanMap; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.internal.util.typedef.internal.U; @@ -104,14 +105,14 @@ public class IgfsMetaUpdatePropertiesProcessor implements EntryProcessor<IgniteU @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException { BinaryRawWriter out = writer.rawWriter(); - out.writeMap(props); + IgfsUtils.writeProperties(out, props); } /** {@inheritDoc} */ @Override public void readBinary(BinaryReader reader) throws BinaryObjectException { BinaryRawReader in = reader.rawReader(); - props = in.readMap(); + props = IgfsUtils.readProperties(in); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java index edec572..5894fa2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java @@ -86,9 +86,6 @@ import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheMode.REPLICATED; import static org.apache.ignite.igfs.IgfsMode.PRIMARY; import static org.apache.ignite.igfs.IgfsMode.PROXY; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME; /** * Test fo regular igfs operations. @@ -803,10 +800,12 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { if (dual) // Check only permissions because user and group will always be present in Hadoop Fs. - assertEquals(props.get(PROP_PERMISSION), igfsSecondary.properties(SUBSUBDIR.toString()).get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfsSecondary.properties(SUBSUBDIR.toString()).get(IgfsUtils.PROP_PERMISSION)); // We check only permission because IGFS client adds username and group name explicitly. - assertEquals(props.get(PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfs.info(SUBSUBDIR).properties().get(IgfsUtils.PROP_PERMISSION)); } /** @@ -824,10 +823,11 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { if (dual) // check permission only since Hadoop Fs will always have user and group: - assertEquals(props.get(PROP_PERMISSION), igfsSecondary.properties(DIR.toString()).get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfsSecondary.properties(DIR.toString()).get(IgfsUtils.PROP_PERMISSION)); // We check only permission because IGFS client adds username and group name explicitly. - assertEquals(props.get(PROP_PERMISSION), igfs.info(DIR).properties().get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), igfs.info(DIR).properties().get(IgfsUtils.PROP_PERMISSION)); } /** @@ -3026,13 +3026,13 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { Map<String, String> props = new HashMap<>(); if (username != null) - props.put(PROP_USER_NAME, username); + props.put(IgfsUtils.PROP_USER_NAME, username); if (grpName != null) - props.put(PROP_GROUP_NAME, grpName); + props.put(IgfsUtils.PROP_GROUP_NAME, grpName); if (perm != null) - props.put(PROP_PERMISSION, perm); + props.put(IgfsUtils.PROP_PERMISSION, perm); return props; } http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java index 6e13280..b8c8978 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java @@ -39,7 +39,6 @@ import java.util.concurrent.CyclicBarrier; import static org.apache.ignite.igfs.IgfsMode.DUAL_ASYNC; import static org.apache.ignite.igfs.IgfsMode.DUAL_SYNC; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION; /** * Tests for IGFS working in mode when remote file system exists: DUAL_SYNC, DUAL_ASYNC. @@ -973,10 +972,12 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest { checkExist(igfs, igfsSecondary, SUBSUBDIR); // Check only permissions because user and group will always be present in Hadoop secondary filesystem. - assertEquals(props.get(PROP_PERMISSION), igfsSecondary.properties(SUBSUBDIR.toString()).get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfsSecondary.properties(SUBSUBDIR.toString()).get(IgfsUtils.PROP_PERMISSION)); // We check only permission because IGFS client adds username and group name explicitly. - assertEquals(props.get(PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfs.info(SUBSUBDIR).properties().get(IgfsUtils.PROP_PERMISSION)); } /** @@ -998,10 +999,12 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest { checkExist(igfs, igfsSecondary, SUBSUBDIR); // Check only permission because in case of Hadoop secondary Fs user and group will always be present: - assertEquals(props.get(PROP_PERMISSION), igfsSecondary.properties(SUBSUBDIR.toString()).get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfsSecondary.properties(SUBSUBDIR.toString()).get(IgfsUtils.PROP_PERMISSION)); // We check only permission because IGFS client adds username and group name explicitly. - assertEquals(props.get(PROP_PERMISSION), igfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION)); + assertEquals(props.get(IgfsUtils.PROP_PERMISSION), + igfs.info(SUBSUBDIR).properties().get(IgfsUtils.PROP_PERMISSION)); } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java index 10208ed..812d48b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java @@ -254,7 +254,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest { IgniteFileSystem fs2 = grid(2).fileSystem("igfs"); try (IgfsOutputStream out = fs0.create(path, 128, false, 1, CFG_GRP_SIZE, - F.asMap(IgfsEx.PROP_PREFER_LOCAL_WRITES, "true"))) { + F.asMap(IgfsUtils.PROP_PREFER_LOCAL_WRITES, "true"))) { // 1.5 blocks byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java index 7a4648a..dfd4611 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java @@ -60,10 +60,6 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.Callable; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME; - /** * Secondary file system which delegates calls to an instance of Hadoop {@link FileSystem}. * <p> @@ -238,8 +234,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys if (perm == null) perm = FsPermission.getDefault(); - return F.asMap(PROP_PERMISSION, String.format("%04o", perm.toShort()), PROP_USER_NAME, status.getOwner(), - PROP_GROUP_NAME, status.getGroup()); + return F.asMap(IgfsUtils.PROP_PERMISSION, String.format("%04o", perm.toShort()), IgfsUtils.PROP_USER_NAME, + status.getOwner(), IgfsUtils.PROP_GROUP_NAME, status.getGroup()); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java index 45b968c..83991aa 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java @@ -86,10 +86,6 @@ import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.P import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH; import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.parameter; import static org.apache.ignite.internal.processors.igfs.IgfsEx.IGFS_SCHEME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PREFER_LOCAL_WRITES; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME; /** * {@code IGFS} Hadoop 1.x file system driver over file system API. To use @@ -509,9 +505,12 @@ public class IgniteHadoopFileSystem extends FileSystem { } secondaryFs.setOwner(toSecondary(p), username, grpName); - } else if (rmtClient.update(convert(p), F.asMap(PROP_USER_NAME, username, PROP_GROUP_NAME, grpName)) == null) + } + else if (rmtClient.update(convert(p), F.asMap(IgfsUtils.PROP_USER_NAME, username, + IgfsUtils.PROP_GROUP_NAME, grpName)) == null) { throw new IOException("Failed to set file permission (file not found?)" + " [path=" + p + ", userName=" + username + ", groupName=" + grpName + ']'); + } } finally { leaveBusy(); @@ -627,7 +626,7 @@ public class IgniteHadoopFileSystem extends FileSystem { else { Map<String,String> propMap = permission(perm); - propMap.put(PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites)); + propMap.put(IgfsUtils.PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites)); // Create stream and close it in the 'finally' section if any sequential operation failed. HadoopIgfsStreamDelegate stream = rmtClient.create(path, overwrite, colocateFileWrites, @@ -1282,8 +1281,8 @@ public class IgniteHadoopFileSystem extends FileSystem { file.modificationTime(), file.accessTime(), permission(file), - file.property(PROP_USER_NAME, user), - file.property(PROP_GROUP_NAME, "users"), + file.property(IgfsUtils.PROP_USER_NAME, user), + file.property(IgfsUtils.PROP_GROUP_NAME, "users"), convert(file.path())) { @Override public String toString() { return "FileStatus [path=" + getPath() + ", isDir=" + isDir() + ", len=" + getLen() + @@ -1302,7 +1301,7 @@ public class IgniteHadoopFileSystem extends FileSystem { if (perm == null) perm = FsPermission.getDefault(); - return F.asMap(PROP_PERMISSION, toString(perm)); + return F.asMap(IgfsUtils.PROP_PERMISSION, toString(perm)); } /** @@ -1320,7 +1319,7 @@ public class IgniteHadoopFileSystem extends FileSystem { * @return Hadoop permission. */ private FsPermission permission(IgfsFile file) { - String perm = file.property(PROP_PERMISSION, null); + String perm = file.property(IgfsUtils.PROP_PERMISSION, null); if (perm == null) return FsPermission.getDefault(); http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java index ac457a4..66255c9 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java @@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.igfs.IgfsHandshakeResponse; import org.apache.ignite.internal.processors.igfs.IgfsModeResolver; import org.apache.ignite.internal.processors.igfs.IgfsPaths; import org.apache.ignite.internal.processors.igfs.IgfsStatus; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.A; @@ -94,10 +95,6 @@ import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.P import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH; import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.parameter; import static org.apache.ignite.internal.processors.igfs.IgfsEx.IGFS_SCHEME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PREFER_LOCAL_WRITES; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME; /** * {@code IGFS} Hadoop 2.x file system driver over file system API. To use @@ -463,9 +460,11 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea try { if (mode(p) == PROXY) secondaryFileSystem().setOwner(toSecondary(p), usr, grp); - else if (rmtClient.update(convert(p), F.asMap(PROP_USER_NAME, usr, PROP_GROUP_NAME, grp)) == null) + else if (rmtClient.update(convert(p), F.asMap(IgfsUtils.PROP_USER_NAME, usr, + IgfsUtils.PROP_GROUP_NAME, grp)) == null) { throw new IOException("Failed to set file permission (file not found?)" + " [path=" + p + ", username=" + usr + ", grpName=" + grp + ']'); + } } finally { leaveBusy(); @@ -579,8 +578,8 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea return os; } else { - Map<String, String> permMap = F.asMap(PROP_PERMISSION, toString(perm), - PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites)); + Map<String, String> permMap = F.asMap(IgfsUtils.PROP_PERMISSION, toString(perm), + IgfsUtils.PROP_PREFER_LOCAL_WRITES, Boolean.toString(preferLocFileWrites)); // Create stream and close it in the 'finally' section if any sequential operation failed. HadoopIgfsStreamDelegate stream; @@ -1002,8 +1001,8 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea file.modificationTime(), file.accessTime(), permission(file), - file.property(PROP_USER_NAME, user), - file.property(PROP_GROUP_NAME, "users"), + file.property(IgfsUtils.PROP_USER_NAME, user), + file.property(IgfsUtils.PROP_GROUP_NAME, "users"), convert(file.path())) { @Override public String toString() { return "FileStatus [path=" + getPath() + ", isDir=" + isDirectory() + ", len=" + getLen() + "]"; @@ -1021,7 +1020,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea if (perm == null) perm = FsPermission.getDefault(); - return F.asMap(PROP_PERMISSION, toString(perm)); + return F.asMap(IgfsUtils.PROP_PERMISSION, toString(perm)); } /** @@ -1039,7 +1038,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea * @return Hadoop permission. */ private FsPermission permission(IgfsFile file) { - String perm = file.property(PROP_PERMISSION, null); + String perm = file.property(IgfsUtils.PROP_PERMISSION, null); if (perm == null) return FsPermission.getDefault(); http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java index c8929a3..90f6bca 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java @@ -20,10 +20,7 @@ package org.apache.ignite.internal.processors.hadoop.igfs; import java.util.Map; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.ignite.IgniteException; - -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION; -import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; /** * Hadoop file system properties. @@ -45,10 +42,10 @@ public class HadoopIgfsProperties { * @throws IgniteException In case of error. */ public HadoopIgfsProperties(Map<String, String> props) throws IgniteException { - usrName = props.get(PROP_USER_NAME); - grpName = props.get(PROP_GROUP_NAME); + usrName = props.get(IgfsUtils.PROP_USER_NAME); + grpName = props.get(IgfsUtils.PROP_GROUP_NAME); - String permStr = props.get(PROP_PERMISSION); + String permStr = props.get(IgfsUtils.PROP_PERMISSION); if (permStr != null) { try { http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java index 5b6fd81..44b8f40 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.ignite.configuration.FileSystemConfiguration; import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory; import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils; -import org.apache.ignite.internal.processors.igfs.IgfsEx; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.processors.igfs.UniversalFileSystemAdapter; /** @@ -84,12 +84,12 @@ public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFile Map<String,String> m = new HashMap<>(3); // max size == 4 - m.put(IgfsEx.PROP_USER_NAME, status.getOwner()); - m.put(IgfsEx.PROP_GROUP_NAME, status.getGroup()); + m.put(IgfsUtils.PROP_USER_NAME, status.getOwner()); + m.put(IgfsUtils.PROP_GROUP_NAME, status.getGroup()); FsPermission perm = status.getPermission(); - m.put(IgfsEx.PROP_PERMISSION, "0" + perm.getUserAction().ordinal() + perm.getGroupAction().ordinal() + + m.put(IgfsUtils.PROP_PERMISSION, "0" + perm.getUserAction().ordinal() + perm.getGroupAction().ordinal() + perm.getOtherAction().ordinal()); return m; http://git-wip-us.apache.org/repos/asf/ignite/blob/e886ad0a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java index 7fd8272..4426847 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopMapReduceTest.java @@ -52,6 +52,7 @@ import org.apache.ignite.internal.processors.hadoop.counter.HadoopPerformanceCou import org.apache.ignite.internal.processors.hadoop.examples.HadoopWordCount1; import org.apache.ignite.internal.processors.hadoop.examples.HadoopWordCount2; import org.apache.ignite.internal.processors.igfs.IgfsEx; +import org.apache.ignite.internal.processors.igfs.IgfsUtils; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.T2; @@ -110,7 +111,7 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest { * @return The owner. */ private static String getOwner(IgfsEx i, IgfsPath p) { - return i.info(p).property(IgfsEx.PROP_USER_NAME); + return i.info(p).property(IgfsUtils.PROP_USER_NAME); } /** @@ -122,7 +123,7 @@ public class HadoopMapReduceTest extends HadoopAbstractWordCountTest { private static String getOwnerSecondary(final IgfsSecondaryFileSystem secFs, final IgfsPath p) { return IgfsUserContext.doAs(USER, new IgniteOutClosure<String>() { @Override public String apply() { - return secFs.info(p).property(IgfsEx.PROP_USER_NAME); + return secFs.info(p).property(IgfsUtils.PROP_USER_NAME); } }); }
