Repository: commons-math Updated Branches: refs/heads/master a94ff90ab -> fd37b5dd0
Reverted flawed fix for MATH-1246. Project: http://git-wip-us.apache.org/repos/asf/commons-math/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-math/commit/fd37b5dd Tree: http://git-wip-us.apache.org/repos/asf/commons-math/tree/fd37b5dd Diff: http://git-wip-us.apache.org/repos/asf/commons-math/diff/fd37b5dd Branch: refs/heads/master Commit: fd37b5dd02bbce93f6f4fceb6bc3e6aa4641c5a7 Parents: a94ff90 Author: Phil Steitz <[email protected]> Authored: Mon Nov 9 15:11:33 2015 -0700 Committer: Phil Steitz <[email protected]> Committed: Mon Nov 9 15:11:33 2015 -0700 ---------------------------------------------------------------------- .../stat/inference/KolmogorovSmirnovTest.java | 61 -------------------- .../inference/KolmogorovSmirnovTestTest.java | 56 ------------------ 2 files changed, 117 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-math/blob/fd37b5dd/src/main/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTest.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTest.java b/src/main/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTest.java index ba18db9..9c8bcb9 100644 --- a/src/main/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTest.java +++ b/src/main/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTest.java @@ -244,9 +244,6 @@ public class KolmogorovSmirnovTest { public double kolmogorovSmirnovTest(double[] x, double[] y, boolean strict) { final long lengthProduct = (long) x.length * y.length; if (lengthProduct < SMALL_SAMPLE_PRODUCT) { - if (hasTies(x, y)) { - return exactP(x, y, strict); - } return integralExactP(integralKolmogorovSmirnovStatistic(x, y) + (strict?1l:0l), x.length, y.length); } if (lengthProduct < LARGE_SAMPLE_PRODUCT) { @@ -976,64 +973,6 @@ public class KolmogorovSmirnovTest { return (double) tail / (double) CombinatoricsUtils.binomialCoefficient(n + m, n); } - /** - * Computes the exact p value for a two-sample Kolmogorov-Smirnov test with - * {@code x} and {@code y} as samples, possibly containing ties. This method - * uses the same implementation as {@link #exactP(double, int, int, boolean)} - * with the exception that it examines partitions of the combined sample, - * preserving ties in the data. What is returned is the exact probability - * that a random partition of the combined dataset into a subset of size - * {@code x.length} and another of size {@code y.length} yields a \(D\) - * value greater than (resp greater than or equal to) \(D(x,y)\). - * <p> - * This method should not be used on large samples (a good rule of thumb is - * to keep the product of the sample sizes less than - * {@link #SMALL_SAMPLE_PRODUCT} when using this method). If the data do - * not contain ties, {@link #exactP(double[], double[], boolean)} should be - * used instead of this method.</p> - * - * @param x first sample - * @param y second sample - * @param strict whether or not the inequality in the null hypothesis is strict - * @return p-value - */ - public double exactP(double[] x, double[] y, boolean strict) { - final long d = integralKolmogorovSmirnovStatistic(x, y); - final int n = x.length; - final int m = y.length; - - // Concatenate x and y into universe, preserving ties in the data - final double[] universe = new double[n + m]; - System.arraycopy(x, 0, universe, 0, n); - System.arraycopy(y, 0, universe, n, m); - - // Iterate over all n, m partitions of the n + m elements in the universe, - // Computing D for each one - Iterator<int[]> combinationsIterator = CombinatoricsUtils.combinationsIterator(n + m, n); - long tail = 0; - final double[] nSet = new double[n]; - final double[] mSet = new double[m]; - while (combinationsIterator.hasNext()) { - // Generate an n-set - final int[] nSetI = combinationsIterator.next(); - // Copy the elements of the universe in the n-set to nSet - // and the others to mSet - int j = 0; - int k = 0; - for (int i = 0; i < n + m; i++) { - if (j < n && nSetI[j] == i) { - nSet[j++] = universe[i]; - } else { - mSet[k++] = universe[i]; - } - } - final long curD = integralKolmogorovSmirnovStatistic(nSet, mSet); - if (curD > d || (curD == d && !strict)) { - tail++; - } - } - return (double) tail / (double) CombinatoricsUtils.binomialCoefficient(n + m, n); - } /** * Uses the Kolmogorov-Smirnov distribution to approximate \(P(D_{n,m} > d)\) where \(D_{n,m}\) http://git-wip-us.apache.org/repos/asf/commons-math/blob/fd37b5dd/src/test/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTestTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTestTest.java b/src/test/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTestTest.java index 2cc1b1d..28db893 100644 --- a/src/test/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTestTest.java +++ b/src/test/java/org/apache/commons/math4/stat/inference/KolmogorovSmirnovTestTest.java @@ -185,62 +185,6 @@ public class KolmogorovSmirnovTestTest { Assert.assertEquals(0.5, test.kolmogorovSmirnovStatistic(smallSample1, smallSample2), TOLERANCE); } - /** Small sample no ties, exactP methods should agree */ - @Test - public void testExactPConsistency() { - final KolmogorovSmirnovTest test = new KolmogorovSmirnovTest(); - final double[] x = { - 1, 7, 9, 13, 19, 21, 22, 23, 24 - }; - final double[] y = { - 3, 4, 12, 16, 20, 27, 28, 32, 44, 54 - }; - Assert.assertEquals(test.exactP(x, y, true), - test.exactP(test.kolmogorovSmirnovStatistic(x, y), - x.length, y.length, true), Double.MIN_VALUE); - Assert.assertEquals(test.exactP(x, y, false), - test.exactP(test.kolmogorovSmirnovStatistic(x, y), - x.length, y.length, false), Double.MIN_VALUE); - } - - /** - * Extreme case for ties - all values the same. Strict p-value should be 0, - * non-strict should be 1 - */ - @Test - public void testExactPNoVariance() { - final KolmogorovSmirnovTest test = new KolmogorovSmirnovTest(); - final double[] x = { - 1, 1, 1, 1, 1, 1 - }; - final double[] y = { - 1, 1, 1, 1 - }; - Assert.assertEquals(0, test.exactP(x, y, true), Double.MIN_VALUE); - Assert.assertEquals(1, test.exactP(x, y, false), Double.MIN_VALUE); - Assert.assertEquals(0, test.kolmogorovSmirnovTest(x, y, true), Double.MIN_VALUE); - Assert.assertEquals(1, test.kolmogorovSmirnovTest(x, y, false), Double.MIN_VALUE); - } - - /** - * Split {0, 0, 0, 1, 1, 1} into 3-sets. Most extreme is 0's vs 1's. Non-strict - * p-value for this split should be 2 / (6 choose 3); strict should be 0. - */ - @Test - public void testExactPSimpleSplit() { - final KolmogorovSmirnovTest test = new KolmogorovSmirnovTest(); - final double[] x = { - 0, 0, 0 - }; - final double[] y = { - 1, 1, 1 - }; - // Above is one way to do this - other way is s/x/y - so 2 in strict test below - Assert.assertEquals(0, test.exactP(x, y, true), Double.MIN_VALUE); - Assert.assertEquals(2 / (double) CombinatoricsUtils.binomialCoefficient(6, 3), - test.exactP(x, y, false), Double.MIN_VALUE); - } - /** * Checks exact p-value computations using critical values from Table 9 in V.K Rohatgi, An * Introduction to Probability and Mathematical Statistics, Wiley, 1976, ISBN 0-471-73135-8.
