[3/4] hadoop git commit: HDFS-8934. Move ShortCircuitShm to hdfs-client. Contributed by Mingliang Liu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95f8e936/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
new file mode 100644
index 000..78325a3
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
@@ -0,0 +1,647 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.shortcircuit;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Random;
+
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdfs.ExtendedBlockId;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.misc.Unsafe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.primitives.Ints;
+
+/**
+ * A shared memory segment used to implement short-circuit reads.
+ */
+public class ShortCircuitShm {
+ private static final Logger LOG =
LoggerFactory.getLogger(ShortCircuitShm.class);
+
+ protected static final int BYTES_PER_SLOT = 64;
+
+ private static final Unsafe unsafe = safetyDance();
+
+ private static Unsafe safetyDance() {
+try {
+ Field f = Unsafe.class.getDeclaredField(theUnsafe);
+ f.setAccessible(true);
+ return (Unsafe)f.get(null);
+} catch (Throwable e) {
+ LOG.error(failed to load misc.Unsafe, e);
+}
+return null;
+ }
+
+ /**
+ * Calculate the usable size of a shared memory segment.
+ * We round down to a multiple of the slot size and do some validation.
+ *
+ * @param stream The stream we're using.
+ * @return The usable size of the shared memory segment.
+ */
+ private static int getUsableLength(FileInputStream stream)
+ throws IOException {
+int intSize = Ints.checkedCast(stream.getChannel().size());
+int slots = intSize / BYTES_PER_SLOT;
+if (slots == 0) {
+ throw new IOException(size of shared memory segment was +
+ intSize + , but that is not enough to hold even one slot.);
+}
+return slots * BYTES_PER_SLOT;
+ }
+
+ /**
+ * Identifies a DfsClientShm.
+ */
+ public static class ShmId implements ComparableShmId {
+private static final Random random = new Random();
+private final long hi;
+private final long lo;
+
+/**
+ * Generate a random ShmId.
+ *
+ * We generate ShmIds randomly to prevent a malicious client from
+ * successfully guessing one and using that to interfere with another
+ * client.
+ */
+public static ShmId createRandom() {
+ return new ShmId(random.nextLong(), random.nextLong());
+}
+
+public ShmId(long hi, long lo) {
+ this.hi = hi;
+ this.lo = lo;
+}
+
+public long getHi() {
+ return hi;
+}
+
+public long getLo() {
+ return lo;
+}
+
+@Override
+public boolean equals(Object o) {
+ if ((o == null) || (o.getClass() != this.getClass())) {
+return false;
+ }
+ ShmId other = (ShmId)o;
+ return new EqualsBuilder().
+ append(hi, other.hi).
+ append(lo, other.lo).
+ isEquals();
+}
+
+@Override
+public int hashCode() {
+ return new HashCodeBuilder().
+ append(this.hi).
+ append(this.lo).
+ toHashCode();
+}
+
+@Override
+public String toString() {
+ return String.format(%016x%016x, hi, lo);
+
[3/4] hadoop git commit: HDFS-8934. Move ShortCircuitShm to hdfs-client. Contributed by Mingliang Liu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/490bb5eb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
new file mode 100644
index 000..78325a3
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
@@ -0,0 +1,647 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.shortcircuit;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Random;
+
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdfs.ExtendedBlockId;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.misc.Unsafe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.primitives.Ints;
+
+/**
+ * A shared memory segment used to implement short-circuit reads.
+ */
+public class ShortCircuitShm {
+ private static final Logger LOG =
LoggerFactory.getLogger(ShortCircuitShm.class);
+
+ protected static final int BYTES_PER_SLOT = 64;
+
+ private static final Unsafe unsafe = safetyDance();
+
+ private static Unsafe safetyDance() {
+try {
+ Field f = Unsafe.class.getDeclaredField(theUnsafe);
+ f.setAccessible(true);
+ return (Unsafe)f.get(null);
+} catch (Throwable e) {
+ LOG.error(failed to load misc.Unsafe, e);
+}
+return null;
+ }
+
+ /**
+ * Calculate the usable size of a shared memory segment.
+ * We round down to a multiple of the slot size and do some validation.
+ *
+ * @param stream The stream we're using.
+ * @return The usable size of the shared memory segment.
+ */
+ private static int getUsableLength(FileInputStream stream)
+ throws IOException {
+int intSize = Ints.checkedCast(stream.getChannel().size());
+int slots = intSize / BYTES_PER_SLOT;
+if (slots == 0) {
+ throw new IOException(size of shared memory segment was +
+ intSize + , but that is not enough to hold even one slot.);
+}
+return slots * BYTES_PER_SLOT;
+ }
+
+ /**
+ * Identifies a DfsClientShm.
+ */
+ public static class ShmId implements ComparableShmId {
+private static final Random random = new Random();
+private final long hi;
+private final long lo;
+
+/**
+ * Generate a random ShmId.
+ *
+ * We generate ShmIds randomly to prevent a malicious client from
+ * successfully guessing one and using that to interfere with another
+ * client.
+ */
+public static ShmId createRandom() {
+ return new ShmId(random.nextLong(), random.nextLong());
+}
+
+public ShmId(long hi, long lo) {
+ this.hi = hi;
+ this.lo = lo;
+}
+
+public long getHi() {
+ return hi;
+}
+
+public long getLo() {
+ return lo;
+}
+
+@Override
+public boolean equals(Object o) {
+ if ((o == null) || (o.getClass() != this.getClass())) {
+return false;
+ }
+ ShmId other = (ShmId)o;
+ return new EqualsBuilder().
+ append(hi, other.hi).
+ append(lo, other.lo).
+ isEquals();
+}
+
+@Override
+public int hashCode() {
+ return new HashCodeBuilder().
+ append(this.hi).
+ append(this.lo).
+ toHashCode();
+}
+
+@Override
+public String toString() {
+ return String.format(%016x%016x, hi, lo);
+
[3/4] hadoop git commit: HDFS-8934. Move ShortCircuitShm to hdfs-client. Contributed by Mingliang Liu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e4afa3a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
new file mode 100644
index 000..78325a3
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java
@@ -0,0 +1,647 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.shortcircuit;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Random;
+
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdfs.ExtendedBlockId;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.io.nativeio.NativeIO.POSIX;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.misc.Unsafe;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.primitives.Ints;
+
+/**
+ * A shared memory segment used to implement short-circuit reads.
+ */
+public class ShortCircuitShm {
+ private static final Logger LOG =
LoggerFactory.getLogger(ShortCircuitShm.class);
+
+ protected static final int BYTES_PER_SLOT = 64;
+
+ private static final Unsafe unsafe = safetyDance();
+
+ private static Unsafe safetyDance() {
+try {
+ Field f = Unsafe.class.getDeclaredField(theUnsafe);
+ f.setAccessible(true);
+ return (Unsafe)f.get(null);
+} catch (Throwable e) {
+ LOG.error(failed to load misc.Unsafe, e);
+}
+return null;
+ }
+
+ /**
+ * Calculate the usable size of a shared memory segment.
+ * We round down to a multiple of the slot size and do some validation.
+ *
+ * @param stream The stream we're using.
+ * @return The usable size of the shared memory segment.
+ */
+ private static int getUsableLength(FileInputStream stream)
+ throws IOException {
+int intSize = Ints.checkedCast(stream.getChannel().size());
+int slots = intSize / BYTES_PER_SLOT;
+if (slots == 0) {
+ throw new IOException(size of shared memory segment was +
+ intSize + , but that is not enough to hold even one slot.);
+}
+return slots * BYTES_PER_SLOT;
+ }
+
+ /**
+ * Identifies a DfsClientShm.
+ */
+ public static class ShmId implements ComparableShmId {
+private static final Random random = new Random();
+private final long hi;
+private final long lo;
+
+/**
+ * Generate a random ShmId.
+ *
+ * We generate ShmIds randomly to prevent a malicious client from
+ * successfully guessing one and using that to interfere with another
+ * client.
+ */
+public static ShmId createRandom() {
+ return new ShmId(random.nextLong(), random.nextLong());
+}
+
+public ShmId(long hi, long lo) {
+ this.hi = hi;
+ this.lo = lo;
+}
+
+public long getHi() {
+ return hi;
+}
+
+public long getLo() {
+ return lo;
+}
+
+@Override
+public boolean equals(Object o) {
+ if ((o == null) || (o.getClass() != this.getClass())) {
+return false;
+ }
+ ShmId other = (ShmId)o;
+ return new EqualsBuilder().
+ append(hi, other.hi).
+ append(lo, other.lo).
+ isEquals();
+}
+
+@Override
+public int hashCode() {
+ return new HashCodeBuilder().
+ append(this.hi).
+ append(this.lo).
+ toHashCode();
+}
+
+@Override
+public String toString() {
+ return String.format(%016x%016x, hi, lo);
+
