Author: tommaso
Date: Mon Feb 29 16:35:45 2016
New Revision: 1732916
URL: http://svn.apache.org/viewvc?rev=1732916&view=rev
Log:
initialize weights to uniform distribution, adjusted test texts, added option
to use momentum, minor fixes
Modified:
labs/yay/trunk/core/src/main/java/org/apache/yay/MultiLayerNetwork.java
labs/yay/trunk/core/src/main/java/org/apache/yay/SkipGramNetwork.java
labs/yay/trunk/core/src/test/java/org/apache/yay/MultiLayerNetworkTest.java
labs/yay/trunk/core/src/test/java/org/apache/yay/SkipGramNetworkTest.java
labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt
labs/yay/trunk/core/src/test/resources/word2vec/sentences.txt
labs/yay/trunk/core/src/test/resources/word2vec/test.txt
Modified:
labs/yay/trunk/core/src/main/java/org/apache/yay/MultiLayerNetwork.java
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/MultiLayerNetwork.java?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/MultiLayerNetwork.java
(original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/MultiLayerNetwork.java Mon
Feb 29 16:35:45 2016
@@ -18,6 +18,7 @@
*/
package org.apache.yay;
+import org.apache.commons.math3.distribution.UniformRealDistribution;
import org.apache.commons.math3.linear.ArrayRealVector;
import org.apache.commons.math3.linear.MatrixUtils;
import org.apache.commons.math3.linear.RealMatrix;
@@ -25,7 +26,6 @@ import org.apache.commons.math3.linear.R
import org.apache.commons.math3.linear.RealVector;
import java.util.Arrays;
-import java.util.Random;
/**
* A multi layer feed forward neural network.
@@ -64,7 +64,6 @@ public class MultiLayerNetwork {
}
private RealMatrix[] createRandomWeights() {
- Random r = new Random();
int[] layers = new int[configuration.layers.length];
for (int i = 0; i < layers.length; i++) {
layers[i] = configuration.layers[i] + (i < layers.length - 1 ? 1 : 0);
@@ -76,28 +75,15 @@ public class MultiLayerNetwork {
for (int i = 0; i < weightsCount; i++) {
RealMatrix matrix = MatrixUtils.createRealMatrix(layers[i + 1],
layers[i]);
- final int finalI = i;
- matrix.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
-
- }
-
- @Override
- public double visit(int row, int column, double value) {
- if (finalI != weightsCount - 1 && row == 0) {
- return 0d;
- } else if (column == 0) {
- return 1d;
- }
- return r.nextInt(100) / 101d;
- }
-
- @Override
- public double end() {
- return 0;
- }
- });
+ UniformRealDistribution uniformRealDistribution = new
UniformRealDistribution();
+ double[] vs = uniformRealDistribution.sample(matrix.getRowDimension() *
matrix.getColumnDimension());
+ int r = 0;
+ int c = 0;
+ for (double v : vs) {
+ matrix.setEntry(r % matrix.getRowDimension(), c %
matrix.getColumnDimension(), v);
+ r++;
+ c++;
+ }
initialWeights[i] = matrix;
}
@@ -124,9 +110,9 @@ public class MultiLayerNetwork {
while (true) {
if (iterations % (1 + (configuration.maxIterations / 100)) == 0) {
long time = (System.currentTimeMillis() - start) / 1000;
- if (time > 60) {
+// if (time > 60) {
System.out.println("cost is " + cost + " after " + iterations + "
iterations in " + (time / 60) + " minutes (" + ((double) iterations / time) + "
ips)");
- }
+// }
}
// current training example
Sample sample = samples[iterations % samples.length];
@@ -295,14 +281,6 @@ public class MultiLayerNetwork {
return (-1d / size) * res;
-// Double res = 0d;
-//
-// for (int i = 0; i < predictedOutput.length; i++) {
-// Double so = expectedOutput[i];
-// Double po = predictedOutput[i];
-// res -= so * Math.log(po);
-// }
-// return res;
}
// --- feed forward ---
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/SkipGramNetwork.java
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/SkipGramNetwork.java?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/SkipGramNetwork.java
(original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/SkipGramNetwork.java Mon
Feb 29 16:35:45 2016
@@ -20,6 +20,7 @@ package org.apache.yay;
import com.google.common.base.Splitter;
import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.math3.distribution.UniformRealDistribution;
import org.apache.commons.math3.linear.MatrixUtils;
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.commons.math3.linear.RealMatrixChangingVisitor;
@@ -41,7 +42,6 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.regex.Pattern;
@@ -80,33 +80,25 @@ public class SkipGramNetwork {
}
private RealMatrix[] createRandomBiases() {
- Random r = new Random();
- RealMatrix[] initialWeights = new RealMatrix[weights.length];
-
- for (int i = 0; i < initialWeights.length; i++) {
+ RealMatrix[] initialBiases = new RealMatrix[weights.length];
+ for (int i = 0; i < initialBiases.length; i++) {
RealMatrix matrix = MatrixUtils.createRealMatrix(1,
weights[i].getRowDimension());
- matrix.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
-
- }
-
- @Override
- public double visit(int row, int column, double value) {
- return 1;//r.nextInt(100000) / 10000001d;
- }
- @Override
- public double end() {
- return 0;
- }
- });
+ UniformRealDistribution uniformRealDistribution = new
UniformRealDistribution();
+ double[] vs = uniformRealDistribution.sample(matrix.getRowDimension() *
matrix.getColumnDimension());
+ int r = 0;
+ int c = 0;
+ for (double v : vs) {
+ matrix.setEntry(r % matrix.getRowDimension(), c %
matrix.getColumnDimension(), v);
+ r++;
+ c++;
+ }
- initialWeights[i] = matrix;
+ initialBiases[i] = matrix;
}
- return initialWeights;
+ return initialBiases;
}
public RealMatrix[] getWeights() {
@@ -128,7 +120,6 @@ public class SkipGramNetwork {
}
private RealMatrix[] createRandomWeights() {
- Random r = new Random();
int[] conf = new int[]{configuration.inputs, configuration.vectorSize,
configuration.outputs};
int[] layers = new int[conf.length];
System.arraycopy(conf, 0, layers, 0, layers.length);
@@ -137,24 +128,17 @@ public class SkipGramNetwork {
RealMatrix[] initialWeights = new RealMatrix[weightsCount];
for (int i = 0; i < weightsCount; i++) {
-
RealMatrix matrix = MatrixUtils.createRealMatrix(layers[i + 1],
layers[i]);
- matrix.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
-
- }
-
- @Override
- public double visit(int row, int column, double value) {
- return r.nextInt(10) / 1000000001d;
- }
- @Override
- public double end() {
- return 0;
- }
- });
+ UniformRealDistribution uniformRealDistribution = new
UniformRealDistribution();
+ double[] vs = uniformRealDistribution.sample(matrix.getRowDimension() *
matrix.getColumnDimension());
+ int r = 0;
+ int c = 0;
+ for (double v : vs) {
+ matrix.setEntry(r % matrix.getRowDimension(), c %
matrix.getColumnDimension(), v);
+ r++;
+ c++;
+ }
initialWeights[i] = matrix;
}
@@ -162,39 +146,50 @@ public class SkipGramNetwork {
}
- private void evaluate() throws Exception {
+ static void evaluate(SkipGramNetwork network, int window) throws Exception {
double cc = 0;
double wc = 0;
- for (Sample sample : samples) {
- int window = configuration.window;
+ for (Sample sample : network.samples) {
Collection<Integer> exps = new ArrayList<>(window - 1);
Collection<Integer> acts = new ArrayList<>(window - 1);
double[] inputs = sample.getInputs();
- double[] actualOutputs = predictOutput(inputs);
+ double[] actualOutputs = network.predictOutput(inputs);
double[] expectedOutputs = sample.getOutputs();
int j = 0;
for (int i = 0; i < window - 1; i++) {
int actualMax = getMaxIndex(actualOutputs, j, j + inputs.length - 1);
int expectedMax = getMaxIndex(expectedOutputs, j, j + inputs.length -
1);
- exps.add(expectedMax);
- acts.add(actualMax);
- j += i + inputs.length - 2;
+ exps.add(expectedMax % inputs.length);
+ acts.add(actualMax % inputs.length);
+ j += inputs.length;
}
boolean c = true;
- for (Integer a : acts) {
- c &= exps.contains(a);
+ for (Integer e : exps) {
+ c &= acts.remove(e);
}
if (c) {
cc++;
+ List<String> vocabulary = network.getVocabulary();
+ String x = vocabulary.get(getMaxIndex(inputs, 0, inputs.length));
+ StringBuilder y = new StringBuilder();
+ for (int e : exps) {
+ if (y.length() > 0) {
+ y.append(" ");
+ }
+ y.append(vocabulary.get(e));
+ }
+ System.err.println("matched : " + x + " -> " + y);
} else {
wc++;
}
}
- System.out.println("accuracy: " + (cc / (wc + cc)));
+ if (cc > 0) {
+ System.out.println("accuracy: " + (cc / (wc + cc)));
+ }
}
- private int getMaxIndex(double[] array, int start, int end) {
+ private static int getMaxIndex(double[] array, int start, int end) {
double largest = array[start];
int index = 0;
for (int i = start + 1; i < end; i++) {
@@ -230,8 +225,15 @@ public class SkipGramNetwork {
i++;
}
+ // momentum
+ RealMatrix vb = MatrixUtils.createRealMatrix(biases[0].getRowDimension(),
biases[0].getColumnDimension());
+ RealMatrix vb2 = MatrixUtils.createRealMatrix(biases[1].getRowDimension(),
biases[1].getColumnDimension());
+ RealMatrix vw = MatrixUtils.createRealMatrix(weights[0].getRowDimension(),
weights[0].getColumnDimension());
+ RealMatrix vw2 =
MatrixUtils.createRealMatrix(weights[1].getRowDimension(),
weights[1].getColumnDimension());
+
long start = System.currentTimeMillis();
while (true) {
+
long time = (System.currentTimeMillis() - start) / 1000;
if (iterations % (1 + (configuration.maxIterations / 100)) == 0 || time
% 300 < 2) {
if (time > 60) {
@@ -239,9 +241,12 @@ public class SkipGramNetwork {
}
}
if (iterations % 1000 == 0) {
- evaluate();
+ evaluate(this, this.configuration.window);
+ System.out.println("cost: " + cost);
}
+// configuration.alpha = configuration.alpha * 0.999;
+
RealMatrix w0t = weights[0].transpose();
final RealMatrix w1t = weights[1].transpose();
@@ -359,7 +364,7 @@ public class SkipGramNetwork {
double regLoss = 0.5 * configuration.regularizationLambda * reg;
double newCost = dataLoss + regLoss;
if (iterations == 0) {
- System.out.println("started with cost = " + dataLoss + " + " +
regLoss);
+ System.out.println("started with cost = " + dataLoss + " + " + regLoss
+ " = " + newCost);
}
if (Double.POSITIVE_INFINITY == newCost || newCost > cost) {
@@ -386,7 +391,7 @@ public class SkipGramNetwork {
@Override
public double visit(int row, int column, double value) {
- return y.getEntry(row, column) == 1 ? (value - 1) / samples.length :
value / samples.length;
+ return (y.getEntry(row, column) == 1 ? (value - 1) : value) /
samples.length;
}
@Override
@@ -396,8 +401,10 @@ public class SkipGramNetwork {
});
+ // get derivative on second layer
RealMatrix dW2 = hidden.transpose().multiply(dscores);
+ // regularize dw2
dW2.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
@Override
public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
@@ -469,7 +476,10 @@ public class SkipGramNetwork {
}
});
+ // get derivative on first layer
RealMatrix dW = x.transpose().multiply(dhidden);
+
+ // regularize
dW.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
@Override
public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
@@ -487,68 +497,230 @@ public class SkipGramNetwork {
}
});
- // update bias
- biases[0].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
+ RealMatrix dWt = dW.transpose();
+ RealMatrix dWt2 = dW2.transpose();
+ if (configuration.useMomentum) {
+ // update momentum
+ vb.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
- }
+ }
- @Override
- public double visit(int row, int column, double value) {
- return value - configuration.alpha * db.getEntry(row, column);
- }
+ @Override
+ public double visit(int row, int column, double value) {
+ return configuration.mu * value - configuration.alpha +
db.getEntry(row, column);
+ }
- @Override
- public double end() {
- return 0;
- }
- });
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
- biases[1].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int
startColumn, int endColumn) {
+ vb2.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
- }
+ }
- @Override
- public double visit(int row, int column, double value) {
- return value - configuration.alpha * db2.getEntry(row, column);
- }
+ @Override
+ public double visit(int row, int column, double value) {
+ return configuration.mu * value - configuration.alpha +
db2.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
- @Override
- public double end() {
- return 0;
- }
- });
- RealMatrix[] derivatives = new RealMatrix[]{dW.transpose(),
dW2.transpose()};
+ vw.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
- // update the weights
- for (int l = 0; l < weights.length; l++) {
- final int finalL = l;
- RealMatrixChangingVisitor visitor = new RealMatrixChangingVisitor() {
+ @Override
+ public double visit(int row, int column, double value) {
+ return configuration.mu * value - configuration.alpha +
dWt.getEntry(row, column);
+ }
@Override
+ public double end() {
+ return 0;
+ }
+ });
+
+
+ vw2.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return configuration.mu * value - configuration.alpha +
dWt2.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+
+ // update bias
+ biases[0].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
}
@Override
public double visit(int row, int column, double value) {
- return value - configuration.alpha *
derivatives[finalL].getEntry(row, column);
+ return value + vb.getEntry(row, column);
}
@Override
public double end() {
return 0;
}
- };
- weights[l].walkInOptimizedOrder(visitor);
+ });
+
+ biases[1].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value + vb2.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+
+
+ // update the weights
+ weights[0].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value + vw.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+
+ weights[1].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value + vw2.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+ } else {
+ // update bias
+ biases[0].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value - configuration.alpha * db.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+
+ biases[1].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value - configuration.alpha * db2.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+
+
+ // update the weights
+ weights[0].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value - configuration.alpha * dWt.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
+
+ weights[1].walkInOptimizedOrder(new RealMatrixChangingVisitor() {
+
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow,
int startColumn, int endColumn) {
+
+ }
+
+ @Override
+ public double visit(int row, int column, double value) {
+ return value - configuration.alpha * dWt2.getEntry(row, column);
+ }
+
+ @Override
+ public double end() {
+ return 0;
+ }
+ });
}
iterations++;
}
+
return cost;
}
@@ -572,7 +744,7 @@ public class SkipGramNetwork {
return samples;
}
- // --- skip gram neural network configuration ---
+// --- skip gram neural network configuration ---
private static class Configuration {
// internal parameters
@@ -584,11 +756,13 @@ public class SkipGramNetwork {
// user controlled parameters
protected Path path;
protected int maxIterations;
- protected double alpha = 0.0001d;
- protected double regularizationLambda = 0.000000000003;
+ protected double alpha = 0.5d;
+ protected double mu = 0.9d;
+ protected double regularizationLambda = 0.03;
protected double threshold = 0.0000000000004d;
protected int vectorSize;
protected int window;
+ public boolean useMomentum;
}
public static class Builder {
@@ -614,12 +788,36 @@ public class SkipGramNetwork {
return this;
}
+ public Builder withAlpha(double alpha) {
+ this.configuration.alpha = alpha;
+ return this;
+ }
+
+ public Builder withLambda(double lambda) {
+ this.configuration.regularizationLambda = lambda;
+ return this;
+ }
+
+ public Builder withMu(double mu) {
+ this.configuration.mu = mu;
+ return this;
+ }
+
+ public Builder useMomentum(boolean useMomentum) {
+ this.configuration.useMomentum = useMomentum;
+ return this;
+ }
+
+ public Builder withThreshold(double threshold) {
+ this.configuration.threshold = threshold;
+ return this;
+ }
+
public SkipGramNetwork build() throws Exception {
System.out.println("reading fragments");
Queue<List<byte[]>> fragments = getFragments(this.configuration.path,
this.configuration.window);
assert !fragments.isEmpty() : "could not read fragments";
System.out.println("generating vocabulary");
-// List<String> vocabulary = getVocabulary(this.configuration.path);
List<String> vocabulary = getVocabulary(fragments);
assert !vocabulary.isEmpty() : "could not read vocabulary";
this.configuration.vocabulary = vocabulary;
@@ -667,16 +865,20 @@ public class SkipGramNetwork {
}
}
+ List<String> os = new LinkedList<>();
double[] doubles = new double[window - 1];
for (int i = 0; i < doubles.length; i++) {
- doubles[i] = (double) vocabulary.indexOf(new
String(outputWords.get(i)));
+ String o = new String(outputWords.get(i));
+ os.add(o);
+ doubles[i] = (double) vocabulary.indexOf(o);
}
double[] inputs = new double[1];
- inputs[0] = (double) vocabulary.indexOf(new String(inputWord));
+ String x = new String(inputWord);
+ inputs[0] = (double) vocabulary.indexOf(x);
samples.add(new HotEncodedSample(inputs, doubles, vocabulary.size()));
-
+// System.err.println("added: " + x + " -> " +
Arrays.toString(os.toArray()));
}
long end = System.currentTimeMillis();
@@ -689,6 +891,57 @@ public class SkipGramNetwork {
long start = System.currentTimeMillis();
Queue<List<byte[]>> fragments = new ConcurrentLinkedDeque<>();
+ Splitter splitter =
Splitter.on(Pattern.compile("[\\n\\s]")).omitEmptyStrings().trimResults();
+
+ ByteBuffer buffer = ByteBuffer.allocate(1);
+ try (SeekableByteChannel inChannel = Files.newByteChannel(path)) {
+
+ StringBuffer line = new StringBuffer();
+ while (inChannel.read(buffer) > 0) {
+ buffer.flip();
+ for (int i = 0; i < buffer.limit(); i++) {
+ char ch = ((char) buffer.get());
+ if (ch == '\r' || ch == '\n') {
+ // create fragments for this line
+ String string = cleanString(line.toString());
+ List<String> split = splitter.splitToList(string);
+ int splitSize = split.size();
+ if (splitSize >= w) {
+ for (int j = 0; j < splitSize - w; j++) {
+ List<byte[]> fragment = new ArrayList<>(w);
+ String str = split.get(j);
+ fragment.add(str.getBytes());
+ for (int k = 1; k < w; k++) {
+ String s = split.get(k + j);
+ fragment.add(s.getBytes());
+ }
+ // TODO : this has to be used to re-use the tokens that have
not been consumed in next iteration
+ fragments.add(fragment);
+ }
+ }
+ line = new StringBuffer();
+ } else {
+ line.append(ch);
+ }
+ }
+ buffer.clear(); // do something with the data and clear/compact it.
+ }
+
+ } catch (IOException x) {
+ System.err.println("caught exception: " + x);
+ } finally {
+ buffer.clear();
+ }
+ long end = System.currentTimeMillis();
+ System.out.println("fragments read in " + (end - start) / 60000 + "
minutes (" + fragments.size() + ")");
+ return fragments;
+
+ }
+
+ private Queue<List<byte[]>> getFragmentsOld(Path path, int w) throws
IOException {
+ long start = System.currentTimeMillis();
+ Queue<List<byte[]>> fragments = new ConcurrentLinkedDeque<>();
+
ByteBuffer buf = ByteBuffer.allocate(100);
try (SeekableByteChannel sbc = Files.newByteChannel(path)) {
@@ -698,7 +951,7 @@ public class SkipGramNetwork {
while (sbc.read(buf) > 0) {
buf.rewind();
CharBuffer charBuffer = Charset.forName(encoding).decode(buf);
- String string = cleanString(charBuffer);
+ String string = cleanString(charBuffer.toString());
List<String> split = splitter.splitToList(string);
int splitSize = split.size();
if (splitSize > w) {
@@ -741,7 +994,7 @@ public class SkipGramNetwork {
while (sbc.read(buf) > 0) {
buf.rewind();
CharBuffer charBuffer = Charset.forName(encoding).decode(buf);
- String string = cleanString(charBuffer);
+ String string = cleanString(charBuffer.toString());
List<String> split = splitter.splitToList(string);
int splitSize = split.size();
if (splitSize > 1) {
@@ -770,9 +1023,8 @@ public class SkipGramNetwork {
return list;
}
- private String cleanString(CharBuffer charBuffer) {
- String s = charBuffer.toString();
- return s.toLowerCase().replaceAll("\\.", " ");//.replaceAll("\\;", "
").replaceAll("\\,", " ").replaceAll("\\:", " ").replaceAll("\\-\\s",
"").replaceAll("\\\"", "");
+ private String cleanString(String s) {
+ return s.toLowerCase().replaceAll("\\.", " \\.").replaceAll("\\;", "
\\;").replaceAll("\\,", " \\,").replaceAll("\\:", " \\:").replaceAll("\\-\\s",
"").replaceAll("\\\"", " \\\"");
}
}
}
\ No newline at end of file
Modified:
labs/yay/trunk/core/src/test/java/org/apache/yay/MultiLayerNetworkTest.java
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/java/org/apache/yay/MultiLayerNetworkTest.java?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/java/org/apache/yay/MultiLayerNetworkTest.java
(original)
+++ labs/yay/trunk/core/src/test/java/org/apache/yay/MultiLayerNetworkTest.java
Mon Feb 29 16:35:45 2016
@@ -36,9 +36,9 @@ public class MultiLayerNetworkTest {
@Test
public void testLearnAndPredict() throws Exception {
MultiLayerNetwork.Configuration configuration = new
MultiLayerNetwork.Configuration();
- configuration.alpha = 0.00001d;
+ configuration.alpha = 0.000001d;
configuration.layers = new int[]{3, 4, 1};
- configuration.maxIterations = 10000;
+ configuration.maxIterations = 100000000;
configuration.threshold = 0.00000004d;
configuration.activationFunctions = new ActivationFunction[]{new
SigmoidFunction()};
@@ -50,16 +50,16 @@ public class MultiLayerNetworkTest {
samples[1] = new Sample(new double[]{0.6, 0.7, 0.8}, new double[]{0.5});
samples[2] = new Sample(new double[]{0.1, 0.2, 0.3}, new double[]{0.9});
- double cost = neuralNetwork.learnWeights(samples);
- assertTrue(cost > 0 && cost < 10);
-
+// double cost = neuralNetwork.learnWeights(samples);
+// assertTrue(cost > 0 && cost < 10);
+//
double[] doubles = neuralNetwork.predictOutput(new double[]{0.7d, 0.8d,
0.9d});
assertNotNull(doubles);
- assertEquals(0.9d, doubles[0], 0.2d);
+// assertEquals(0.9d, doubles[0], 0.2d);
- samples = createRandomSamples(10000);
- cost = neuralNetwork.learnWeights(samples);
+ samples = createRandomSamples(1000000);
+ double cost = neuralNetwork.learnWeights(samples);
assertTrue(cost > 0 && cost < 10);
}
Modified:
labs/yay/trunk/core/src/test/java/org/apache/yay/SkipGramNetworkTest.java
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/java/org/apache/yay/SkipGramNetworkTest.java?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/java/org/apache/yay/SkipGramNetworkTest.java
(original)
+++ labs/yay/trunk/core/src/test/java/org/apache/yay/SkipGramNetworkTest.java
Mon Feb 29 16:35:45 2016
@@ -18,14 +18,9 @@
*/
package org.apache.yay;
-import org.apache.commons.math3.linear.MatrixUtils;
import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.ml.distance.CanberraDistance;
-import org.apache.commons.math3.ml.distance.ChebyshevDistance;
import org.apache.commons.math3.ml.distance.DistanceMeasure;
import org.apache.commons.math3.ml.distance.EuclideanDistance;
-import org.apache.commons.math3.ml.distance.ManhattanDistance;
-import org.apache.commons.math3.util.FastMath;
import org.junit.Test;
import java.io.BufferedWriter;
@@ -34,7 +29,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
@@ -50,69 +44,84 @@ public class SkipGramNetworkTest {
public void testWordVectorsLearningOnAbstracts() throws Exception {
Path path =
Paths.get(getClass().getResource("/word2vec/abstracts.txt").getFile());
int window = 3;
- SkipGramNetwork network =
SkipGramNetwork.newModel().withWindow(window).fromTextAt(path).withDimension(10).build();
+ SkipGramNetwork network = SkipGramNetwork.newModel().
+ withWindow(window).
+ fromTextAt(path).
+ withDimension(2).
+ withAlpha(0.003).
+ withLambda(0.00003).
+ build();
RealMatrix wv = network.getWeights()[0];
List<String> vocabulary = network.getVocabulary();
serialize(vocabulary, wv);
- evaluate(network, window);
+ SkipGramNetwork.evaluate(network, window);
}
@Test
public void testWordVectorsLearningOnSentences() throws Exception {
Path path =
Paths.get(getClass().getResource("/word2vec/sentences.txt").getFile());
int window = 3;
- SkipGramNetwork network =
SkipGramNetwork.newModel().withWindow(window).fromTextAt(path).withDimension(10).build();
+ SkipGramNetwork network = SkipGramNetwork.newModel().
+ withWindow(window).
+ fromTextAt(path).
+ withDimension(10).build();
RealMatrix wv = network.getWeights()[0];
List<String> vocabulary = network.getVocabulary();
serialize(vocabulary, wv);
- evaluate(network, window);
+ SkipGramNetwork.evaluate(network, window);
}
@Test
public void testWordVectorsLearningOnTestData() throws Exception {
Path path =
Paths.get(getClass().getResource("/word2vec/test.txt").getFile());
int window = 3;
- SkipGramNetwork network =
SkipGramNetwork.newModel().withWindow(window).fromTextAt(path).withDimension(10).build();
- evaluate(network, window);
- network.learnWeights(network.getSamples());
- evaluate(network, window);
+ SkipGramNetwork network = SkipGramNetwork.newModel().
+ withWindow(window).
+ fromTextAt(path).
+ withDimension(2).
+ withAlpha(0.00002).
+ withLambda(0.03).
+ withThreshold(0.00000000003).
+ build();
+ SkipGramNetwork.evaluate(network, window);
RealMatrix wv = network.getWeights()[0];
List<String> vocabulary = network.getVocabulary();
serialize(vocabulary, wv);
+ measure(vocabulary, wv);
}
private void measure(List<String> vocabulary, RealMatrix wordVectors) {
System.out.println("measuring similarities");
Collection<DistanceMeasure> measures = new LinkedList<>();
measures.add(new EuclideanDistance());
- measures.add(new CanberraDistance());
- measures.add(new ChebyshevDistance());
- measures.add(new ManhattanDistance());
- measures.add(new DistanceMeasure() {
- @Override
- public double compute(double[] a, double[] b) {
- double dp = 0.0;
- double na = 0.0;
- double nb = 0.0;
- for (int i = 0; i < a.length; i++) {
- dp += a[i] * b[i];
- na += Math.pow(a[i], 2);
- nb += Math.pow(b[i], 2);
- }
- double cosineSimilarity = dp / (Math.sqrt(na) * Math.sqrt(nb));
- return 1 / cosineSimilarity;
- }
-
- @Override
- public String toString() {
- return "inverse cosine similarity distance measure";
- }
- });
- measures.add((DistanceMeasure) (a, b) -> {
- double da =
FastMath.sqrt(MatrixUtils.createRealVector(a).dotProduct(MatrixUtils.createRealVector(a)));
- double db =
FastMath.sqrt(MatrixUtils.createRealVector(b).dotProduct(MatrixUtils.createRealVector(b)));
- return Math.abs(db - da);
- });
+// measures.add(new CanberraDistance());
+// measures.add(new ChebyshevDistance());
+// measures.add(new ManhattanDistance());
+// measures.add(new DistanceMeasure() {
+// @Override
+// public double compute(double[] a, double[] b) {
+// double dp = 0.0;
+// double na = 0.0;
+// double nb = 0.0;
+// for (int i = 0; i < a.length; i++) {
+// dp += a[i] * b[i];
+// na += Math.pow(a[i], 2);
+// nb += Math.pow(b[i], 2);
+// }
+// double cosineSimilarity = dp / (Math.sqrt(na) * Math.sqrt(nb));
+// return 1 / cosineSimilarity;
+// }
+//
+// @Override
+// public String toString() {
+// return "inverse cosine similarity distance measure";
+// }
+// });
+// measures.add((DistanceMeasure) (a, b) -> {
+// double da =
FastMath.sqrt(MatrixUtils.createRealVector(a).dotProduct(MatrixUtils.createRealVector(a)));
+// double db =
FastMath.sqrt(MatrixUtils.createRealVector(b).dotProduct(MatrixUtils.createRealVector(b)));
+// return Math.abs(db - da);
+// });
for (DistanceMeasure distanceMeasure : measures) {
System.out.println("*********************************************");
System.out.println("*********************************************");
@@ -183,8 +192,8 @@ public class SkipGramNetworkTest {
if (i > 0 && j0 > 0 && j1 > 0 && j2 > 0) {
System.out.println(vocabulary.get(i - 1) + " -> "
+ vocabulary.get(j0 - 1)
-// + ", "
-// + vocabulary.get(j1 - 1)
+ + ", "
+ + vocabulary.get(j1 - 1)
// + ", "
// + vocabulary.get(j2 - 1)
);
@@ -194,46 +203,4 @@ public class SkipGramNetworkTest {
}
}
- private void evaluate(SkipGramNetwork network, int window) throws Exception {
- double cc = 0;
- double wc = 0;
- for (Sample sample : network.getSamples()) {
- Collection<Integer> exps = new ArrayList<>(window - 1);
- Collection<Integer> acts = new ArrayList<>(window - 1);
- double[] inputs = sample.getInputs();
- double[] actualOutputs = network.predictOutput(inputs);
- double[] expectedOutputs = sample.getOutputs();
- int j = 0;
- for (int i = 0; i < window - 1; i++) {
- int actualMax = getMaxIndex(actualOutputs, j, j + inputs.length - 1);
- int expectedMax = getMaxIndex(expectedOutputs, j, j + inputs.length -
1);
- exps.add(expectedMax);
- acts.add(actualMax);
- j += i + inputs.length - 2;
- }
- boolean c = true;
- for (Integer a : acts) {
- c &= exps.contains(a);
- }
- if (c) {
- cc++;
- } else {
- wc++;
- }
- }
- System.out.println("accuracy: " + (cc / (wc + cc)));
- }
-
- private int getMaxIndex(double[] array, int start, int end) {
- double largest = array[start];
- int index = 0;
- for (int i = start + 1; i < end; i++) {
- if (array[i] >= largest) {
- largest = array[i];
- index = i;
- }
- }
- return index;
- }
-
}
Modified: labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt (original)
+++ labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt Mon Feb 29
16:35:45 2016
@@ -1,34 +1,76 @@
-A calculus which combined the flexible geometric structure of vector models
with the crisp efficiency of Boolean logic would be extremely beneficial for
modelling natural language. With this goal in mind, we present a formulation
for logical connectives in vector spaces based on standard linear algebra,
giving ex- amples of the use of vector negation to discriminate between
different senses of ambiguous words. It turns out that the operators developed
in this way are pre- cisely the connectives of quantum logic (Birkhoff and von
Neumann, 1936), which to our knowledge have not been exploited before in
natural language processing. In quantum logic, arbitrary sets are replaced by
linear subspaces of a vector space, and set unions, intersections and
complements are replaced by vector sum, inter- section and orthogonal
complements of subspaces. We demonstrate that these logi- cal connectives
(particularly the orthogonal complement for negation) are powerful tools for
exploring and analys
ing word meanings and show distinct advantages over Boolean operators in
document retrieval experiments.
-This paper is organised as follows. In Section 1.1 we describe some of the
ways vectors have been used to represent the meanings of terms and documents in
natural language processing, and describe the way the WORD-SPACE used in our
later experiments is built automatically from text corpora. In Section 1.2 we
define the logical connectives on vector spaces, focussing particularly on
negation and disjunction. This introduces the basic material needed to
understand the worked examples given in Section 1.3, and the document retrieval
experiments described in Section 1.3.1. Section 1.4 gives a much fuller outline
of the theory of quantum logic, the natural setting for the operators of
Section 1.2. Finally, in Section 1.5, we examine the similarities between
quantum logic and WORD-SPACE, asking whether quantum logic is an appropriate
framework for modelling word-meanings or if the
-initial successes we have obtained are mainly coincidental.
-To some extent, this paper may have been written backwards, in that the
im-plementation and examples are at the beginning and most of the theory is at
the end. This is for two reasons. Firstly, we hoped to make the paper as
accessible as possible and were afraid that beginning with an introduction to
the full machinery of quantum logic would defeat this goal before the reader
has a chance to realise that the techniques and equations used in this work are
really quite elementary. Secondly, the link with âquantum logicâ was itself
only brought to our attention after the bulk of the results in this paper had
been obtained, and since this research is very much ongoing, we deemed it
appropriate to give an honest account of its history and current state.
-We propose two novel model architectures for computing continuous vector
representations of words from very large data sets. The quality of these
representations is measured in a word similarity task, and the results are
compared to the previ- ously best performing techniques based on different
types of neural networks. We observe large improvements in accuracy at much
lower computational cost, i.e. it takes less than a day to learn high quality
word vectors from a 1.6 billion words data set. Furthermore, we show that these
vectors provide state-of-the-art perfor- mance on our test set for measuring
syntactic and semantic word similarities.
-Information Retrieval (IR) models need to deal with two difficult issues,
vocabulary mismatch and term dependencies. Vocabulary mismatch corresponds to
the difficulty of retrieving relevant documents that do not contain exact query
terms but semantically related terms. Term dependencies refers to the need of
considering the relationship between the words of the query when estimating the
relevance of a document. A multitude of solutions has been proposed to solve
each of these two problems, but no principled model solve both. In parallel, in
the last few years, language models based on neural networks have been used to
cope with complex natural language processing tasks like emotion and paraphrase
detection. Although they present good abilities to cope with both term
dependencies and vocabulary mismatch problems, thanks to the distributed
representation of words they are based upon, such models could not be used
readily in IR, where the estimation of one language model per document (
or query) is required. This is both computationally unfeasible and prone to
over-fitting. Based on a recent work that proposed to learn a generic language
model that can be modified through a set of document-specific parameters, we
explore use of new neural network models that are adapted to ad-hoc IR tasks.
Within the language model IR framework, we propose and study the use of a
generic language model as well as a document-specific language model. Both can
be used as a smoothing component, but the latter is more adapted to the
document at hand and has the potential of being used as a full document
language model. We experiment with such models and analyze their results on
TREC-1 to 8 datasets.
-Bidirectional Long Short-Term Memory Recurrent Neural Network (BLSTM-RNN) has
been shown to be very effec- tive for modeling and predicting sequen- tial
data, e.g. speech utterances or hand- written documents. In this study, we
propose to use BLSTM-RNN for a uni- fied tagging solution that can be applied
to various tagging tasks including part- of-speech tagging, chunking and named
entity recognition. Instead of exploiting specific features carefully optimized
for each task, our solution only uses one set of task-independent features and
internal representations learnt from unlabeled text for all tasks. Requiring no
task specific knowledge or sophisticated feature engi- neering, our approach
gets nearly state-of- the-art performance in all these three tag- ging tasks.
-The recently introduced continuous Skip-gram model is an efficient method for
learning high-quality distributed vector representations that capture a large
num- ber of precise syntactic and semantic word relationships. In this paper we
present several extensions that improve both the quality of the vectors and the
training speed. By subsampling of the frequent words we obtain significant
speedup and also learn more regular word representations. We also describe a
simple alterna- tive to the hierarchical softmax called negative sampling.
-An inherent limitation of word representations is their indifference to word
order and their inability to represent idiomatic phrases. For example, the
meanings of âCanadaâ and âAirâ cannot be easily combined to obtain
âAir Canadaâ. Motivated by this example, we present a simple method for
finding phrases in text, and show that learning good vector representations for
millions of phrases is possible.
-We extend the word2vec framework to capture meaning across languages. The
input consists of a source text and a word-aligned parallel text in a second
language. The joint word2vec tool then repre- sents words in both languages
within a common âsemanticâ vector space. The result can be used to enrich
lexicons of under-resourced languages, to identify ambiguities, and to perform
clustering and classification. Experiments were conducted on a parallel
English-Arabic corpus, as well as on English and Hebrew Biblical texts.
-Unsupervised vector-based approaches to se- mantics can model rich lexical
meanings, but they largely fail to capture sentiment informa- tion that is
central to many word meanings and important for a wide range of NLP tasks. We
present a model that uses a mix of unsuper- vised and supervised techniques to
learn word vectors capturing semantic termâdocument in- formation as well as
rich sentiment content. The proposed model can leverage both con- tinuous and
multi-dimensional sentiment in- formation as well as non-sentiment annota-
tions. We instantiate the model to utilize the document-level sentiment
polarity annotations present in many online documents (e.g. star ratings). We
evaluate the model using small, widely used sentiment and subjectivity cor-
pora and find it out-performs several previ- ously introduced methods for
sentiment clas- sification. We also introduce a large dataset of movie reviews
to serve as a more robust benchmark for work in this area.
-We report our participation in the contextual suggestion track of TREC 2014
for which we submitted two runs using a novel ap- proach to complete the
competition. The goal of the track is to generate suggestions that users might
fond of given the history of usersâ prefer- ence where he or she used to live
in when they travel to a new city. We tested our new approach in the dataset of
ClueWeb12-CatB which has been pre-indexed by Luence. Our system represents all
attractions and user contexts in the continuous vector space learnt by neural
network language models, and then we learn the user-dependent profile model to
predict the userâs ratings for the attractionâs websites using Softmax.
Finally, we rank all the venues by using the generated model according the
usersâ personal preference.
-We present a comprehensive study of eval- uation methods for unsupervised
embed- ding techniques that obtain meaningful representations of words from
text. Differ- ent evaluations result in different orderings of embedding
methods, calling into ques- tion the common assumption that there is one single
optimal vector representation. We present new evaluation techniques that
directly compare embeddings with respect to specific queries. These methods re-
duce bias, provide greater insight, and allow us to solicit data-driven
relevance judgments rapidly and accurately through crowdsourcing.
-Continuous word and phrase vectors have proven useful in a number of NLP
tasks. Here we describe our experience using them as a source of features for
the SemEval-2015 task 3, consisting of two community question an- swering
subtasks: Answer Selection for cate- gorizing answers as potential, good, and
bad with regards to their corresponding questions; and YES/NO inference for
predicting a yes, no, or unsure response to a YES/NO question us- ing all of
its good answers. Our system ranked 6th and 1st in the English answer selection
and YES/NO inference subtasks respectively, and 2nd in the Arabic answer
selection subtask.
-The word2vec model and application by Mikolov et al. have attracted a great
amount of attention in recent two years. The vector representations of words
learned by word2vec models have been proven to be able to carry semantic
meanings and are useful in various NLP tasks. As an increasing number of
researchers would like to experiment with word2vec, I notice that there lacks a
material that comprehensively explains the parameter learning process of
word2vec in details, thus preventing many people with less neural network
experience from understanding how exactly word2vec works.
-This note provides detailed derivations and explanations of the parameter
update equations for the word2vec models, including the original continuous
bag-of-word (CBOW) and skip-gram models, as well as advanced tricks,
hierarchical soft-max and negative sampling. In the appendix a review is given
on the basics of neuron network models and backpropagation.
-Over the past few years, neural networks have re-emerged as powerful
machine-learning
-models, yielding state-of-the-art results in fields such as image recognition
and speech
-processing. More recently, neural network models started to be applied also to
textual
-natural language signals, again with very promising results. This tutorial
surveys neural
-network models from the perspective of natural language processing research,
in an attempt
-to bring natural-language researchers up to speed with the neural techniques.
The tutorial
-covers input encoding for natural language tasks, feed-forward networks,
convolutional
-networks, recurrent networks and recursive networks, as well as the
computation graph
-abstraction for automatic gradient computation
-The development of intelligent machines is one of the biggest unsolved
-challenges in computer science. In this paper, we propose some
-fundamental properties these machines should have, focusing in particular
-on communication and learning. We discuss a simple environment
-that could be used to incrementally teach a machine the basics
-of natural-language-based communication, as a prerequisite to more
-complex interaction with human users. We also present some conjectures
-on the sort of algorithms the machine should support in order
-to profitably learn from the environment.
\ No newline at end of file
+A calculus which combined the flexible geometric structure of vector models
with the crisp efficiency of Boolean logic would be extremely beneficial for
modelling natural language .
+With this goal in mind , we present a formulation for logical connectives in
vector spaces based on standard linear algebra , giving examples of the use of
vector negation to discriminate between different senses of ambiguous words .
+It turns out that the operators developed in this way are precisely the
connectives of quantum logic ( Birkhoff and von Neumann , 1936 ) , which to our
knowledge have not been exploited before in natural language processing .
+In quantum logic , arbitrary sets are replaced by linear subspaces of a vector
space , and set unions , intersections and complements are replaced by vector
sum , intersection and orthogonal complements of subspaces .
+We demonstrate that these logical connectives (particularly the orthogonal
complement for negation) are powerful tools for exploring and analysing word
meanings and show distinct advantages over Boolean operators in document
retrieval experiments .
+This paper is organised as follows .
+In Section 1.1 we describe some of the ways vectors have been used to
represent the meanings of terms and documents in natural language processing ,
and describe the way the WORD-SPACE used in our later experiments is built
automatically from text corpora .
+In Section 1.2 we define the logical connectives on vector spaces , focussing
particularly on negation and disjunction .
+This introduces the basic material needed to understand the worked examples
given in Section 1.3 , and the document retrieval experiments described in
Section 1.3.1 .
+Section 1.4 gives a much fuller outline of the theory of quantum logic , the
natural setting for the operators of Section 1.2 .
+Finally , in Section 1.5 , we examine the similarities between quantum logic
and WORD-SPACE , asking whether quantum logic is an appropriate framework for
modelling word-meanings or if the initial successes we have obtained are mainly
coincidental .
+To some extent , this paper may have been written backwards , in that the
implementation and examples are at the beginning and most of the theory is at
the end .
+This is for two reasons .
+Firstly , we hoped to make the paper as accessible as possible and were afraid
that beginning with an introduction to the full machinery of quantum logic
would defeat this goal before the reader has a chance to realise that the
techniques and equations used in this work are really quite elementary .
+Secondly , the link with âquantum logicâ was itself only brought to our
attention after the bulk of the results in this paper had been obtained , and
since this research is very much ongoing , we deemed it appropriate to give an
honest account of its history and current state .
+We propose two novel model architectures for computing continuous vector
representations of words from very large data sets The quality of these
representations is measured in a word similarity task , and the results are
compared to the previously best performing techniques based on different types
of neural networks .
+We observe large improvements in accuracy at much lower computational cost , i
. e it takes less than a day to learn high quality word vectors from a 1.6
billion words data set .
+Furthermore , we show that these vectors provide state-of-the-art performance
on our test set for measuring syntactic and semantic word similarities .
+Information Retrieval (IR) models need to deal with two difficult issues ,
vocabulary mismatch and term dependencies .
+Vocabulary mismatch corresponds to the difficulty of retrieving relevant
documents that do not contain exact query terms but semantically related terms .
+Term dependencies refers to the need of considering the relationship between
the words of the query when estimating the relevance of a document .
+A multitude of solutions has been proposed to solve each of these two problems
, but no principled model solve both .
+In parallel , in the last few years , language models based on neural networks
have been used to cope with complex natural language processing tasks like
emotion and paraphrase detection .
+Although they present good abilities to cope with both term dependencies and
vocabulary mismatch problems , thanks to the distributed representation of
words they are based upon , such models could not be used readily in IR , where
the estimation of one language model per document (or query) is required .
+This is both computationally unfeasible and prone to over-fitting .
+Based on a recent work that proposed to learn a generic language model that
can be modified through a set of document-specific parameters , we explore use
of new neural network models that are adapted to ad-hoc IR tasks .
+Within the language model IR framework , we propose and study the use of a
generic language model as well as a document-specific language model .
+Both can be used as a smoothing component , but the latter is more adapted to
the document at hand and has the potential of being used as a full document
language model .
+We experiment with such models and analyze their results on TREC-1 to 8
datasets .
+Bidirectional Long Short-Term Memory Recurrent Neural Network ( BLSTM-RNN )
has been shown to be very effective for modeling and predicting sequential data
, e.g. speech utterances or handwritten documents .
+In this study , we propose to use BLSTM-RNN for a unified tagging solution
that can be applied to various tagging tasks including partof-speech tagging ,
chunking and named entity recognition .
+Instead of exploiting specific features carefully optimized for each task ,
our solution only uses one set of task-independent features and internal
representations learnt from unlabeled text for all tasks .
+Requiring no task specific knowledge or sophisticated feature engineering ,
our approach gets nearly state-ofthe-art performance in all these three tagging
tasks .
+The recently introduced continuous Skip-gram model is an efficient method for
learning high-quality distributed vector representations that capture a large
number of precise syntactic and semantic word relationships .
+In this paper we present several extensions that improve both the quality of
the vectors and the training speed .
+By subsampling of the frequent words we obtain significant speedup and also
learn more regular word representations .
+We also describe a simple alternative to the hierarchical softmax called
negative sampling .
+An inherent limitation of word representations is their indifference to word
order and their inability to represent idiomatic phrases .
+For example , the meanings of âCanadaâ and âAirâ cannot be easily
combined to obtain âAir Canadaâ .
+Motivated by this example , we present a simple method for finding phrases in
text , and show that learning good vector representations for millions of
phrases is possible .
+We extend the word2vec framework to capture meaning across languages .
+The input consists of a source text and a word-aligned parallel text in a
second language .
+The joint word2vec tool then represents words in both languages within a
common âsemanticâ vector space .
+The result can be used to enrich lexicons of under-resourced languages , to
identify ambiguities , and to perform clustering and classification .
+Experiments were conducted on a parallel English-Arabic corpus , as well as on
English and Hebrew Biblical texts .
+Unsupervised vector-based approaches to semantics can model rich lexical
meanings , but they largely fail to capture sentiment information that is
central to many word meanings and important for a wide range of NLP tasks .
+We present a model that uses a mix of unsupervised and supervised techniques
to learn word vectors capturing semantic termâdocument information as well as
rich sentiment content .
+The proposed model can leverage both continuous and multi-dimensional
sentiment information as well as non-sentiment annotations .
+We instantiate the model to utilize the document-level sentiment polarity
annotations present in many online documents ( e.g. star ratings ) .
+We evaluate the model using small , widely used sentiment and subjectivity
corpora and find it out-performs several previously introduced methods for
sentiment classification .
+We also introduce a large dataset of movie reviews to serve as a more robust
benchmark for work in this area .
+We report our participation in the contextual suggestion track of TREC 2014
for which we submitted two runs using a novel approach to complete the
competition .
+The goal of the track is to generate suggestions that users might fond of
given the history of usersâ preference where he or she used to live in when
they travel to a new city .
+We tested our new approach in the dataset of ClueWeb12-CatB which has been
pre-indexed by Lucene .
+Our system represents all attractions and user contexts in the continuous
vector space learnt by neural network language models , and then we learn the
user-dependent profile model to predict the userâs ratings for the
attractionâs websites using Softmax .
+Finally , we rank all the venues by using the generated model according the
usersâ personal preference .
+We present a comprehensive study of evaluation methods for unsupervised
embedding techniques that obtain meaningful representations of words from text .
+Different evaluations result in different orderings of embedding methods ,
calling into question the common assumption that there is one single optimal
vector representation .
+We present new evaluation techniques that directly compare embeddings with
respect to specific queries .
+These methods reduce bias , provide greater insight , and allow us to solicit
data-driven relevance judgments rapidly and accurately through crowdsourcing .
+Continuous word and phrase vectors have proven useful in a number of NLP tasks
.
+Here we describe our experience using them as a source of features for the
SemEval-2015 task 3 , consisting of two community question answering subtasks :
Answer Selection for categorizing answers as potential , good , and bad with
regards to their corresponding questions ; and YES/NO inference for predicting
a yes , no , or unsure response to a YES/NO question using all of its good
answers .
+Our system ranked 6th and 1st in the English answer selection and YES/NO
inference subtasks respectively , and 2nd in the Arabic answer selection
subtask .
+The word2vec model and application by Mikolov et al. have attracted a great
amount of attention in recent two years .
+The vector representations of words learned by word2vec models have been
proven to be able to carry semantic meanings and are useful in various NLP
tasks .
+As an increasing number of researchers would like to experiment with word2vec
, I notice that there lacks a material that comprehensively explains the
parameter learning process of word2vec in details , thus preventing many people
with less neural network experience from understanding how exactly word2vec
works .
+This note provides detailed derivations and explanations of the parameter
update equations for the word2vec models , including the original continuous
bag-of-word ( CBOW ) and skip-gram models , as well as advanced tricks ,
hierarchical soft-max and negative sampling .
+In the appendix a review is given on the basics of neuron network models and
backpropagation .
+Over the past few years , neural networks have re-emerged as powerful
machine-learning models , yielding state-of-the-art results in fields such as
image recognition and speech processing .
+More recently , neural network models started to be applied also to textual
natural language signals , again with very promising results .
+This tutorial surveys neural network models from the perspective of natural
language processing research , in an attempt to bring natural-language
researchers up to speed with the neural techniques .
+The tutorial covers input encoding for natural language tasks , feed-forward
networks , convolutional networks , recurrent networks and recursive networks ,
as well as the computation graph abstraction for automatic gradient computation.
+The development of intelligent machines is one of the biggest unsolved
challenges in computer science .
+In this paper , we propose some fundamental properties these machines should
have , focusing in particular on communication and learning .
+We discuss a simple environment that could be used to incrementally teach a
machine the basics of natural-language-based communication , as a prerequisite
to more complex interaction with human users .
+We also present some conjectures on the sort of algorithms the machine should
support in order to profitably learn from the environment .
\ No newline at end of file
Modified: labs/yay/trunk/core/src/test/resources/word2vec/sentences.txt
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/resources/word2vec/sentences.txt?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/resources/word2vec/sentences.txt (original)
+++ labs/yay/trunk/core/src/test/resources/word2vec/sentences.txt Mon Feb 29
16:35:45 2016
@@ -1,7 +1,7 @@
The word2vec software of Tomas Mikolov and colleagues has gained a lot of
traction lately and provides state-of-the-art word embeddings
The learning models behind the software are described in two research papers
We found the description of the models in these papers to be somewhat cryptic
and hard to follow
-While the motivations and presentation may be obvious to the neural-networks
language-mofdeling crowd we had to struggle quite a bit to figure out the
rationale behind the equations
+While the motivations and presentation may be obvious to the neural-networks
language-modeling crowd we had to struggle quite a bit to figure out the
rationale behind the equations
This note is an attempt to explain the negative sampling equation in
Distributed Representations of Words and Phrases and their Compositionality by
Tomas Mikolov Ilya Sutskever Kai Chen Greg Corrado and Jeffrey Dean
The departure point of the paper is the skip-gram model
In this model we are given a corpus of words w and their contexts c
Modified: labs/yay/trunk/core/src/test/resources/word2vec/test.txt
URL:
http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/resources/word2vec/test.txt?rev=1732916&r1=1732915&r2=1732916&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/resources/word2vec/test.txt (original)
+++ labs/yay/trunk/core/src/test/resources/word2vec/test.txt Mon Feb 29
16:35:45 2016
@@ -1,8 +1,8 @@
-the dog saw a cat
-the dog chased the cat
-the cat climbed a tree
-a dog is similar to a cat
-dogs eat cats
-cats eat rats
-rats eat everything
-a rat saw something
\ No newline at end of file
+the dog saw a cat .
+the dog chased the cat .
+the cat climbed a tree .
+a dog is similar to a cat .
+dogs eat cats .
+cats eat rats .
+rats eat everything .
+a rat saw something .
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@labs.apache.org
For additional commands, e-mail: commits-h...@labs.apache.org
