Repository: incubator-hivemall Updated Branches: refs/heads/master 3a718713a -> d902dc257
Fix typo: Statefull -> Stateful Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/d902dc25 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/d902dc25 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/d902dc25 Branch: refs/heads/master Commit: d902dc25777ef5be37d7394bf57acf84d6f2836d Parents: 3a71871 Author: Takuya Kitazawa <tak...@apache.org> Authored: Wed Feb 21 06:05:44 2018 +0900 Committer: Takuya Kitazawa <tak...@apache.org> Committed: Wed Feb 21 06:05:44 2018 +0900 ---------------------------------------------------------------------- .../java/hivemall/GeneralLearnerBaseUDTF.java | 12 +-- .../hivemall/fm/FactorizationMachineUDTF.java | 14 ++-- .../main/java/hivemall/recommend/SlimUDTF.java | 12 +-- .../ProbabilisticTopicModelBaseUDTF.java | 12 +-- .../hivemall/utils/io/NioStatefulSegment.java | 80 ++++++++++++++++++++ .../hivemall/utils/io/NioStatefullSegment.java | 80 -------------------- 6 files changed, 105 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java index f1bc045..2651da2 100644 --- a/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java +++ b/core/src/main/java/hivemall/GeneralLearnerBaseUDTF.java @@ -34,7 +34,7 @@ import hivemall.utils.collections.IMapIterator; import hivemall.utils.hadoop.HiveUtils; import hivemall.utils.io.FileUtils; import hivemall.utils.io.NIOUtils; -import hivemall.utils.io.NioStatefullSegment; +import hivemall.utils.io.NioStatefulSegment; import hivemall.utils.lang.FloatAccumulator; import hivemall.utils.lang.NumberUtils; import hivemall.utils.lang.Primitives; @@ -106,7 +106,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF { // for iterations @Nullable - protected transient NioStatefullSegment fileIO; + protected transient NioStatefulSegment fileIO; @Nullable protected transient ByteBuffer inputBuf; private int iterations; @@ -304,7 +304,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF { } ByteBuffer buf = inputBuf; - NioStatefullSegment dst = fileIO; + NioStatefulSegment dst = fileIO; if (buf == null) { final File file; @@ -322,7 +322,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF { throw new UDFArgumentException(e); } this.inputBuf = buf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MB - this.fileIO = dst = new NioStatefullSegment(file, false); + this.fileIO = dst = new NioStatefulSegment(file, false); } int featureVectorBytes = 0; @@ -416,7 +416,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF { return featureVector; } - private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefullSegment dst) + private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefulSegment dst) throws HiveException { srcBuf.flip(); try { @@ -536,7 +536,7 @@ public abstract class GeneralLearnerBaseUDTF extends LearnerBaseUDTF { protected final void runIterativeTraining(@Nonnegative final int iterations) throws HiveException { final ByteBuffer buf = this.inputBuf; - final NioStatefullSegment dst = this.fileIO; + final NioStatefulSegment dst = this.fileIO; assert (buf != null); assert (dst != null); final long numTrainingExamples = count; http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java b/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java index e1982a8..cb72d0f 100644 --- a/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java +++ b/core/src/main/java/hivemall/fm/FactorizationMachineUDTF.java @@ -29,7 +29,7 @@ import hivemall.optimizer.LossFunctions.LossType; import hivemall.utils.collections.Fastutil; import hivemall.utils.hadoop.HiveUtils; import hivemall.utils.io.FileUtils; -import hivemall.utils.io.NioStatefullSegment; +import hivemall.utils.io.NioStatefulSegment; import hivemall.utils.lang.NumberUtils; import hivemall.utils.lang.SizeOf; import hivemall.utils.math.MathUtils; @@ -111,7 +111,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions { // file IO private ByteBuffer _inputBuf; - private NioStatefullSegment _fileIO; + private NioStatefulSegment _fileIO; @Override protected Options getOptions() { @@ -304,7 +304,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions { } ByteBuffer inputBuf = _inputBuf; - NioStatefullSegment dst = _fileIO; + NioStatefulSegment dst = _fileIO; if (inputBuf == null) { final File file; try { @@ -322,7 +322,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions { } this._inputBuf = inputBuf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MiB - this._fileIO = dst = new NioStatefullSegment(file, false); + this._fileIO = dst = new NioStatefulSegment(file, false); } int xBytes = Feature.requiredBytes(x); @@ -341,7 +341,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions { inputBuf.putDouble(y); } - private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefullSegment dst) + private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefulSegment dst) throws HiveException { srcBuf.flip(); try { @@ -404,7 +404,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions { /** * Update regularization parameters `lambda` as follows: - * + * * <pre> * grad_lambdaw0 = (grad l(p,y)) * (-2 * alpha * w_0) * grad_lambdawg = (grad l(p,y)) * (-2 * alpha * (\sum_{l \in group(g)} x_l * w_l)) @@ -536,7 +536,7 @@ public class FactorizationMachineUDTF extends UDTFWithOptions { protected void runTrainingIteration(int iterations) throws HiveException { final ByteBuffer inputBuf = this._inputBuf; - final NioStatefullSegment fileIO = this._fileIO; + final NioStatefulSegment fileIO = this._fileIO; assert (inputBuf != null); assert (fileIO != null); final long numTrainingExamples = _t; http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/recommend/SlimUDTF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/recommend/SlimUDTF.java b/core/src/main/java/hivemall/recommend/SlimUDTF.java index 1689744..7977a38 100644 --- a/core/src/main/java/hivemall/recommend/SlimUDTF.java +++ b/core/src/main/java/hivemall/recommend/SlimUDTF.java @@ -27,7 +27,7 @@ import hivemall.math.vector.VectorProcedure; import hivemall.utils.collections.Fastutil; import hivemall.utils.hadoop.HiveUtils; import hivemall.utils.io.FileUtils; -import hivemall.utils.io.NioStatefullSegment; +import hivemall.utils.io.NioStatefulSegment; import hivemall.utils.lang.NumberUtils; import hivemall.utils.lang.Primitives; import hivemall.utils.lang.SizeOf; @@ -139,7 +139,7 @@ public class SlimUDTF extends UDTFWithOptions { private transient FloatMatrix _dataMatrix; // used to store KNN data into temporary file for iterative training - private transient NioStatefullSegment _fileIO; + private transient NioStatefulSegment _fileIO; private transient ByteBuffer _inputBuf; private ConversionState _cvState; @@ -302,7 +302,7 @@ public class SlimUDTF extends UDTFWithOptions { @Nonnull final Int2ObjectMap<Int2FloatMap> knnItems, final int numKNNItems) throws HiveException { ByteBuffer buf = this._inputBuf; - NioStatefullSegment dst = this._fileIO; + NioStatefulSegment dst = this._fileIO; if (buf == null) { // invoke only at task node (initialize is also invoked in compilation) @@ -319,7 +319,7 @@ public class SlimUDTF extends UDTFWithOptions { } this._inputBuf = buf = ByteBuffer.allocateDirect(8 * 1024 * 1024); // 8MB - this._fileIO = dst = new NioStatefullSegment(file, false); + this._fileIO = dst = new NioStatefulSegment(file, false); } int recordBytes = SizeOf.INT + SizeOf.INT + SizeOf.INT * 2 * knnItems.size() @@ -349,7 +349,7 @@ public class SlimUDTF extends UDTFWithOptions { } private static void writeBuffer(@Nonnull final ByteBuffer srcBuf, - @Nonnull final NioStatefullSegment dst) throws HiveException { + @Nonnull final NioStatefulSegment dst) throws HiveException { srcBuf.flip(); try { dst.write(srcBuf); @@ -488,7 +488,7 @@ public class SlimUDTF extends UDTFWithOptions { private void runIterativeTraining() throws HiveException { final ByteBuffer buf = this._inputBuf; - final NioStatefullSegment dst = this._fileIO; + final NioStatefulSegment dst = this._fileIO; assert (buf != null); assert (dst != null); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java b/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java index c3dab89..b740d47 100644 --- a/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java +++ b/core/src/main/java/hivemall/topicmodel/ProbabilisticTopicModelBaseUDTF.java @@ -23,7 +23,7 @@ import hivemall.annotations.VisibleForTesting; import hivemall.utils.hadoop.HiveUtils; import hivemall.utils.io.FileUtils; import hivemall.utils.io.NIOUtils; -import hivemall.utils.io.NioStatefullSegment; +import hivemall.utils.io.NioStatefulSegment; import hivemall.utils.lang.NumberUtils; import hivemall.utils.lang.Primitives; import hivemall.utils.lang.SizeOf; @@ -76,7 +76,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions { protected ListObjectInspector wordCountsOI; // for iterations - protected NioStatefullSegment fileIO; + protected NioStatefulSegment fileIO; protected ByteBuffer inputBuf; private float cumPerplexity; @@ -188,7 +188,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions { } ByteBuffer buf = inputBuf; - NioStatefullSegment dst = fileIO; + NioStatefulSegment dst = fileIO; if (buf == null) { final File file; @@ -206,7 +206,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions { throw new UDFArgumentException(e); } this.inputBuf = buf = ByteBuffer.allocateDirect(1024 * 1024); // 1 MB - this.fileIO = dst = new NioStatefullSegment(file, false); + this.fileIO = dst = new NioStatefulSegment(file, false); } // wordCounts length, wc1 length, wc1 string, wc2 length, wc2 string, ... @@ -254,7 +254,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions { miniBatchCount = 0; } - private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefullSegment dst) + private static void writeBuffer(@Nonnull ByteBuffer srcBuf, @Nonnull NioStatefulSegment dst) throws HiveException { srcBuf.flip(); try { @@ -289,7 +289,7 @@ public abstract class ProbabilisticTopicModelBaseUDTF extends UDTFWithOptions { protected final void runIterativeTraining(@Nonnegative final int iterations) throws HiveException { final ByteBuffer buf = this.inputBuf; - final NioStatefullSegment dst = this.fileIO; + final NioStatefulSegment dst = this.fileIO; assert (buf != null); assert (dst != null); final long numTrainingExamples = model.getDocCount(); http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java b/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java new file mode 100644 index 0000000..95111c5 --- /dev/null +++ b/core/src/main/java/hivemall/utils/io/NioStatefulSegment.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.utils.io; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; + +import javax.annotation.Nonnull; +import javax.annotation.concurrent.NotThreadSafe; + +@NotThreadSafe +public final class NioStatefulSegment extends NioSegment { + + private long curPos; + + public NioStatefulSegment(File file) { + this(file, false); + } + + public NioStatefulSegment(File file, boolean readOnly) { + super(file, readOnly); + this.curPos = 0L; + } + + public void resetPosition() { + this.curPos = 0L; + } + + public long getPosition() { + return curPos; + } + + public void setPosition(long pos) { + this.curPos = pos; + } + + public int read(@Nonnull ByteBuffer buf) throws IOException { + int bytes = super.read(curPos, buf); + this.curPos += bytes; + return bytes; + } + + public int write(@Nonnull ByteBuffer buf) throws IOException { + int bytes = super.write(curPos, buf); + this.curPos += bytes; + return bytes; + } + + @Override + public int read(long filePos, @Nonnull ByteBuffer buf) throws IOException { + int bytes = super.read(filePos, buf); + this.curPos = filePos + bytes; + return bytes; + } + + @Override + public int write(long filePos, @Nonnull ByteBuffer buf) throws IOException { + int bytes = super.write(filePos, buf); + this.curPos = filePos + bytes; + return bytes; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/d902dc25/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java b/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java deleted file mode 100644 index 5074dca..0000000 --- a/core/src/main/java/hivemall/utils/io/NioStatefullSegment.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package hivemall.utils.io; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import javax.annotation.Nonnull; -import javax.annotation.concurrent.NotThreadSafe; - -@NotThreadSafe -public final class NioStatefullSegment extends NioSegment { - - private long curPos; - - public NioStatefullSegment(File file) { - this(file, false); - } - - public NioStatefullSegment(File file, boolean readOnly) { - super(file, readOnly); - this.curPos = 0L; - } - - public void resetPosition() { - this.curPos = 0L; - } - - public long getPosition() { - return curPos; - } - - public void setPosition(long pos) { - this.curPos = pos; - } - - public int read(@Nonnull ByteBuffer buf) throws IOException { - int bytes = super.read(curPos, buf); - this.curPos += bytes; - return bytes; - } - - public int write(@Nonnull ByteBuffer buf) throws IOException { - int bytes = super.write(curPos, buf); - this.curPos += bytes; - return bytes; - } - - @Override - public int read(long filePos, @Nonnull ByteBuffer buf) throws IOException { - int bytes = super.read(filePos, buf); - this.curPos = filePos + bytes; - return bytes; - } - - @Override - public int write(long filePos, @Nonnull ByteBuffer buf) throws IOException { - int bytes = super.write(filePos, buf); - this.curPos = filePos + bytes; - return bytes; - } - -}