HADOOP-15146. Remove DataOutputByteBuffer. Contributed by BELUGA BEHR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dde15790 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dde15790 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dde15790 Branch: refs/heads/HDFS-12943 Commit: dde1579096aa53f0dbb133bcecb64733c9e5b5d6 Parents: a0bde7d Author: Steve Loughran <ste...@apache.org> Authored: Mon Apr 2 15:52:40 2018 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Apr 2 15:52:40 2018 +0100 ---------------------------------------------------------------------- .../apache/hadoop/io/DataOutputByteBuffer.java | 137 ------------- .../apache/hadoop/io/TestDataByteBuffers.java | 196 ++++++------------- .../org/apache/hadoop/mapreduce/TestTaskID.java | 21 +- 3 files changed, 74 insertions(+), 280 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dde15790/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java deleted file mode 100644 index b85f693..0000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.io; - -import java.io.DataOutputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.ArrayList; -import java.util.LinkedList; - -public class DataOutputByteBuffer extends DataOutputStream { - - static class Buffer extends OutputStream { - - final byte[] b = new byte[1]; - final boolean direct; - final List<ByteBuffer> active = new ArrayList<ByteBuffer>(); - final List<ByteBuffer> inactive = new LinkedList<ByteBuffer>(); - int size; - int length; - ByteBuffer current; - - Buffer(int size, boolean direct) { - this.direct = direct; - this.size = size; - current = direct - ? ByteBuffer.allocateDirect(size) - : ByteBuffer.allocate(size); - } - @Override - public void write(int b) { - this.b[0] = (byte)(b & 0xFF); - write(this.b); - } - @Override - public void write(byte[] b) { - write(b, 0, b.length); - } - @Override - public void write(byte[] b, int off, int len) { - int rem = current.remaining(); - while (len > rem) { - current.put(b, off, rem); - length += rem; - current.flip(); - active.add(current); - off += rem; - len -= rem; - rem = getBuffer(len); - } - current.put(b, off, len); - length += len; - } - int getBuffer(int newsize) { - if (inactive.isEmpty()) { - size = Math.max(size << 1, newsize); - current = direct - ? ByteBuffer.allocateDirect(size) - : ByteBuffer.allocate(size); - } else { - current = inactive.remove(0); - } - return current.remaining(); - } - ByteBuffer[] getData() { - ByteBuffer[] ret = active.toArray(new ByteBuffer[active.size() + 1]); - ByteBuffer tmp = current.duplicate(); - tmp.flip(); - ret[ret.length - 1] = tmp.slice(); - return ret; - } - int getLength() { - return length; - } - void reset() { - length = 0; - current.rewind(); - inactive.add(0, current); - for (int i = active.size() - 1; i >= 0; --i) { - ByteBuffer b = active.remove(i); - b.rewind(); - inactive.add(0, b); - } - current = inactive.remove(0); - } - } - - private final Buffer buffers; - - public DataOutputByteBuffer() { - this(32); - } - - public DataOutputByteBuffer(int size) { - this(size, false); - } - - public DataOutputByteBuffer(int size, boolean direct) { - this(new Buffer(size, direct)); - } - - private DataOutputByteBuffer(Buffer buffers) { - super(buffers); - this.buffers = buffers; - } - - public ByteBuffer[] getData() { - return buffers.getData(); - } - - public int getLength() { - return buffers.getLength(); - } - - public void reset() { - this.written = 0; - buffers.reset(); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/dde15790/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java index 1a9dd3b..d06ebaf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java @@ -22,7 +22,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.Random; import org.junit.Test; @@ -30,53 +29,67 @@ import static org.junit.Assert.*; public class TestDataByteBuffers { - private static void readJunk(DataInput in, Random r, long seed, int iter) + private static final Random RAND = new Random(31L); + + private static void readJunk(DataInput in, int iter) throws IOException { - r.setSeed(seed); + RAND.setSeed(31L); for (int i = 0; i < iter; ++i) { - switch (r.nextInt(7)) { - case 0: - assertEquals((byte)(r.nextInt() & 0xFF), in.readByte()); break; - case 1: - assertEquals((short)(r.nextInt() & 0xFFFF), in.readShort()); break; - case 2: - assertEquals(r.nextInt(), in.readInt()); break; - case 3: - assertEquals(r.nextLong(), in.readLong()); break; - case 4: - assertEquals(Double.doubleToLongBits(r.nextDouble()), - Double.doubleToLongBits(in.readDouble())); break; - case 5: - assertEquals(Float.floatToIntBits(r.nextFloat()), - Float.floatToIntBits(in.readFloat())); break; - case 6: - int len = r.nextInt(1024); - byte[] vb = new byte[len]; - r.nextBytes(vb); - byte[] b = new byte[len]; - in.readFully(b, 0, len); - assertArrayEquals(vb, b); - break; + switch (RAND.nextInt(7)) { + case 0: + assertEquals((byte)(RAND.nextInt() & 0xFF), in.readByte()); break; + case 1: + assertEquals((short)(RAND.nextInt() & 0xFFFF), in.readShort()); break; + case 2: + assertEquals(RAND.nextInt(), in.readInt()); break; + case 3: + assertEquals(RAND.nextLong(), in.readLong()); break; + case 4: + assertEquals(Double.doubleToLongBits(RAND.nextDouble()), + Double.doubleToLongBits(in.readDouble())); + break; + case 5: + assertEquals(Float.floatToIntBits(RAND.nextFloat()), + Float.floatToIntBits(in.readFloat())); + break; + case 6: + int len = RAND.nextInt(1024); + byte[] vb = new byte[len]; + RAND.nextBytes(vb); + byte[] b = new byte[len]; + in.readFully(b, 0, len); + assertArrayEquals(vb, b); + break; + default: + throw new IOException(); } } } - private static void writeJunk(DataOutput out, Random r, long seed, int iter) - throws IOException { - r.setSeed(seed); + private static void writeJunk(DataOutput out, int iter) + throws IOException { + RAND.setSeed(31L); for (int i = 0; i < iter; ++i) { - switch (r.nextInt(7)) { - case 0: out.writeByte(r.nextInt()); break; - case 1: out.writeShort((short)(r.nextInt() & 0xFFFF)); break; - case 2: out.writeInt(r.nextInt()); break; - case 3: out.writeLong(r.nextLong()); break; - case 4: out.writeDouble(r.nextDouble()); break; - case 5: out.writeFloat(r.nextFloat()); break; - case 6: - byte[] b = new byte[r.nextInt(1024)]; - r.nextBytes(b); - out.write(b); - break; + switch (RAND.nextInt(7)) { + case 0: + out.writeByte(RAND.nextInt()); break; + case 1: + out.writeShort((short)(RAND.nextInt() & 0xFFFF)); break; + case 2: + out.writeInt(RAND.nextInt()); break; + case 3: + out.writeLong(RAND.nextLong()); break; + case 4: + out.writeDouble(RAND.nextDouble()); break; + case 5: + out.writeFloat(RAND.nextFloat()); break; + case 6: + byte[] b = new byte[RAND.nextInt(1024)]; + RAND.nextBytes(b); + out.write(b); + break; + default: + throw new IOException(); } } } @@ -84,113 +97,26 @@ public class TestDataByteBuffers { @Test public void testBaseBuffers() throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("SEED: " + seed); - writeJunk(dob, r, seed, 1000); + writeJunk(dob, 1000); DataInputBuffer dib = new DataInputBuffer(); dib.reset(dob.getData(), 0, dob.getLength()); - readJunk(dib, r, seed, 1000); + readJunk(dib, 1000); dob.reset(); - writeJunk(dob, r, seed, 1000); + writeJunk(dob, 1000); dib.reset(dob.getData(), 0, dob.getLength()); - readJunk(dib, r, seed, 1000); - } - - @Test - public void testByteBuffers() throws IOException { - DataOutputByteBuffer dob = new DataOutputByteBuffer(); - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("SEED: " + seed); - writeJunk(dob, r, seed, 1000); - DataInputByteBuffer dib = new DataInputByteBuffer(); - dib.reset(dob.getData()); - readJunk(dib, r, seed, 1000); - - dob.reset(); - writeJunk(dob, r, seed, 1000); - dib.reset(dob.getData()); - readJunk(dib, r, seed, 1000); - } - - private static byte[] toBytes(ByteBuffer[] bufs, int len) { - byte[] ret = new byte[len]; - int pos = 0; - for (int i = 0; i < bufs.length; ++i) { - int rem = bufs[i].remaining(); - bufs[i].get(ret, pos, rem); - pos += rem; - } - return ret; - } - - @Test - public void testDataOutputByteBufferCompatibility() throws IOException { - DataOutputBuffer dob = new DataOutputBuffer(); - DataOutputByteBuffer dobb = new DataOutputByteBuffer(); - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("SEED: " + seed); - writeJunk(dob, r, seed, 1000); - writeJunk(dobb, r, seed, 1000); - byte[] check = toBytes(dobb.getData(), dobb.getLength()); - assertEquals(check.length, dob.getLength()); - assertArrayEquals(check, Arrays.copyOf(dob.getData(), dob.getLength())); - - dob.reset(); - dobb.reset(); - writeJunk(dob, r, seed, 3000); - writeJunk(dobb, r, seed, 3000); - check = toBytes(dobb.getData(), dobb.getLength()); - assertEquals(check.length, dob.getLength()); - assertArrayEquals(check, Arrays.copyOf(dob.getData(), dob.getLength())); - - dob.reset(); - dobb.reset(); - writeJunk(dob, r, seed, 1000); - writeJunk(dobb, r, seed, 1000); - check = toBytes(dobb.getData(), dobb.getLength()); - assertEquals("Failed Checking length = " + check.length, - check.length, dob.getLength()); - assertArrayEquals(check, Arrays.copyOf(dob.getData(), dob.getLength())); + readJunk(dib, 1000); } @Test - public void TestDataInputByteBufferCompatibility() throws IOException { + public void testDataInputByteBufferCompatibility() throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("SEED: " + seed); - writeJunk(dob, r, seed, 1000); + writeJunk(dob, 1000); ByteBuffer buf = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); DataInputByteBuffer dib = new DataInputByteBuffer(); dib.reset(buf); - readJunk(dib, r, seed, 1000); - } - - @Test - public void TestDataOutputByteBufferCompatibility() throws IOException { - DataOutputByteBuffer dob = new DataOutputByteBuffer(); - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("SEED: " + seed); - writeJunk(dob, r, seed, 1000); - ByteBuffer buf = ByteBuffer.allocate(dob.getLength()); - for (ByteBuffer b : dob.getData()) { - buf.put(b); - } - buf.flip(); - DataInputBuffer dib = new DataInputBuffer(); - dib.reset(buf.array(), 0, buf.remaining()); - readJunk(dib, r, seed, 1000); + readJunk(dib, 1000); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/dde15790/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java index 5531074..f8b6c0c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java @@ -17,8 +17,11 @@ */ package org.apache.hadoop.mapreduce; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; + import org.apache.hadoop.io.DataInputByteBuffer; -import org.apache.hadoop.io.DataOutputByteBuffer; import org.apache.hadoop.io.WritableUtils; import org.junit.Test; import static org.junit.Assert.*; @@ -73,7 +76,7 @@ public class TestTaskID { * Test of getTaskType method, of class TaskID. */ @Test - public void testGetTaskType_0args() { + public void testGetTaskType0args() { JobID jobId = new JobID("1234", 0); for (TaskType type : TaskType.values()) { @@ -253,17 +256,18 @@ public class TestTaskID { */ @Test public void testReadFields() throws Exception { - DataOutputByteBuffer out = new DataOutputByteBuffer(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); out.writeInt(0); out.writeInt(1); WritableUtils.writeVInt(out, 4); - out.write(new byte[] { 0x31, 0x32, 0x33, 0x34}); + out.write(new byte[] {0x31, 0x32, 0x33, 0x34}); WritableUtils.writeEnum(out, TaskType.REDUCE); DataInputByteBuffer in = new DataInputByteBuffer(); - in.reset(out.getData()); + in.reset(ByteBuffer.wrap(baos.toByteArray())); TaskID instance = new TaskID(); @@ -280,14 +284,15 @@ public class TestTaskID { public void testWrite() throws Exception { JobID jobId = new JobID("1234", 1); TaskID taskId = new TaskID(jobId, TaskType.JOB_SETUP, 0); - DataOutputByteBuffer out = new DataOutputByteBuffer(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); taskId.write(out); DataInputByteBuffer in = new DataInputByteBuffer(); byte[] buffer = new byte[4]; - in.reset(out.getData()); + in.reset(ByteBuffer.wrap(baos.toByteArray())); assertEquals("The write() method did not write the expected task ID", 0, in.readInt()); @@ -430,7 +435,7 @@ public class TestTaskID { * Test of getTaskType method, of class TaskID. */ @Test - public void testGetTaskType_char() { + public void testGetTaskTypeChar() { assertEquals("The getTaskType() method did not return the expected type", TaskType.MAP, TaskID.getTaskType('m')); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
