[
https://issues.apache.org/jira/browse/SPARK-3987?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Debasish Das reopened SPARK-3987:
---------------------------------
I can send you a further list of failures...this is one more example...I
strongly suggest moving to a robust convergence criteria inside NNLS than
adding hacks on step sizes...
P = [0.986619, 0.639909, 0.748906, 0.900377, 0.688079, 0.734711, 0.835164,
0.723973, 0.822436, 0.852591, 0.699979, 0.609533, 0.559504, 0.708015, 0.544744,
0.658359, 0.632510, 0.751316, 0.653993, 0.642734, 0.799106, 0.898689, 0.712825,
0.878405, 0.849565; 0.639909, 1.055175, 0.884940, 0.975502, 0.815121, 0.845699,
0.899780, 0.709264, 0.960949, 1.021108, 0.896508, 0.692635, 0.659746, 0.809355,
0.539466, 0.730501, 0.639971, 0.881502, 0.840159, 0.628515, 0.917052, 0.950677,
0.823301, 1.022355, 0.994935; 0.748906, 0.884940, 1.421868, 1.175203, 0.986093,
1.028669, 1.091437, 0.849192, 1.204776, 1.249037, 1.115160, 0.815680, 0.772715,
0.971263, 0.621197, 0.875925, 0.757031, 1.034483, 1.022001, 0.705077, 1.115237,
1.164796, 0.983688, 1.226847, 1.180170; 0.900377, 0.975502, 1.175203, 1.676666,
1.065369, 1.109756, 1.181625, 0.944176, 1.220418, 1.328803, 1.156144, 0.984967,
0.916500, 1.046903, 0.728221, 0.991042, 0.855095, 1.181719, 1.095485, 0.901193,
1.214851, 1.277434, 1.077374, 1.372354, 1.356724; 0.688079, 0.815121, 0.986093,
1.065369, 1.249908, 0.953824, 1.004765, 0.771115, 1.082255, 1.161322, 1.021400,
0.757474, 0.736266, 0.923406, 0.598005, 0.812629, 0.706870, 1.011984, 0.968135,
0.682813, 1.034818, 1.039625, 0.937088, 1.152792, 1.121475; 0.734711, 0.845699,
1.028669, 1.109756, 0.953824, 1.334869, 1.091564, 0.824709, 1.157992, 1.226413,
1.045522, 0.731169, 0.709382, 0.980030, 0.634635, 0.853988, 0.758256, 1.070744,
0.997542, 0.692832, 1.118828, 1.119519, 0.977618, 1.202464, 1.152186; 0.835164,
0.899780, 1.091437, 1.181625, 1.004765, 1.091564, 1.566492, 0.994066, 1.307932,
1.301575, 1.075206, 0.697477, 0.670553, 1.068449, 0.725819, 0.908391, 0.877151,
1.083915, 0.993647, 0.735423, 1.202928, 1.275592, 1.040266, 1.242863, 1.150213;
0.723973, 0.709264, 0.849192, 0.944176, 0.771115, 0.824709, 0.994066, 1.291185,
1.084907, 0.934737, 0.802436, 0.559117, 0.510659, 0.801938, 0.624240, 0.682461,
0.761152, 0.633965, 0.633248, 0.622034, 0.826798, 1.084260, 0.777897, 0.868127,
0.764337; 0.822436, 0.960949, 1.204776, 1.220418, 1.082255, 1.157992, 1.307932,
1.084907, 1.821667, 1.386725, 1.220398, 0.710929, 0.683088, 1.116057, 0.719380,
0.925482, 0.907590, 1.010205, 1.041999, 0.673723, 1.224967, 1.373886, 1.082813,
1.261720, 1.130003; 0.852591, 1.021108, 1.249037, 1.328803, 1.161322, 1.226413,
1.301575, 0.934737, 1.386725, 1.831419, 1.289097, 0.885418, 0.868685, 1.190024,
0.740180, 1.034238, 0.884778, 1.342867, 1.252762, 0.811797, 1.374303, 1.319088,
1.190385, 1.478896, 1.427420; 0.699979, 0.896508, 1.115160, 1.156144, 1.021400,
1.045522, 1.075206, 0.802436, 1.220398, 1.289097, 1.501713, 0.831799, 0.807718,
0.994071, 0.599941, 0.876646, 0.730630, 1.074102, 1.088886, 0.677923, 1.126406,
1.127440, 1.013104, 1.257194, 1.217591; 0.609533, 0.692635, 0.815680, 0.984967,
0.757474, 0.731169, 0.697477, 0.559117, 0.710929, 0.885418, 0.831799, 1.206659,
0.820404, 0.673313, 0.477307, 0.698907, 0.529141, 0.853290, 0.812957, 0.721698,
0.775190, 0.790003, 0.746242, 0.994816, 1.053857; 0.559504, 0.659746, 0.772715,
0.916500, 0.736266, 0.709382, 0.670553, 0.510659, 0.683088, 0.868685, 0.807718,
0.820404, 1.103586, 0.666515, 0.455429, 0.667077, 0.500250, 0.855895, 0.807716,
0.674035, 0.759131, 0.732206, 0.732658, 0.965557, 1.021876; 0.708015, 0.809355,
0.971263, 1.046903, 0.923406, 0.980030, 1.068449, 0.801938, 1.116057, 1.190024,
0.994071, 0.673313, 0.666515, 1.293629, 0.631919, 0.822011, 0.745899, 1.060809,
0.965440, 0.672388, 1.090059, 1.069719, 0.959136, 1.162012, 1.109763; 0.544744,
0.539466, 0.621197, 0.728221, 0.598005, 0.634635, 0.725819, 0.624240, 0.719380,
0.740180, 0.599941, 0.477307, 0.455429, 0.631919, 0.802572, 0.551340, 0.549393,
0.654652, 0.567237, 0.531461, 0.684655, 0.749979, 0.627718, 0.744096, 0.710310;
0.658359, 0.730501, 0.875925, 0.991042, 0.812629, 0.853988, 0.908391, 0.682461,
0.925482, 1.034238, 0.876646, 0.698907, 0.667077, 0.822011, 0.551340, 1.075722,
0.642962, 0.955470, 0.862026, 0.658120, 0.955639, 0.944118, 0.835790, 1.057347,
1.042521; 0.632510, 0.639971, 0.757031, 0.855095, 0.706870, 0.758256, 0.877151,
0.761152, 0.907590, 0.884778, 0.730630, 0.529141, 0.500250, 0.745899, 0.549393,
0.642962, 0.976258, 0.726041, 0.658693, 0.579842, 0.811285, 0.917944, 0.731996,
0.859613, 0.798469; 0.751316, 0.881502, 1.034483, 1.181719, 1.011984, 1.070744,
1.083915, 0.633965, 1.010205, 1.342867, 1.074102, 0.853290, 0.855895, 1.060809,
0.654652, 0.955470, 0.726041, 1.779551, 1.210339, 0.817870, 1.289494, 1.030990,
1.082950, 1.419900, 1.451896; 0.653993, 0.840159, 1.022001, 1.095485, 0.968135,
0.997542, 0.993647, 0.633248, 1.041999, 1.252762, 1.088886, 0.812957, 0.807716,
0.965440, 0.567237, 0.862026, 0.658693, 1.210339, 1.442263, 0.688333, 1.139661,
0.988895, 0.991869, 1.276105, 1.281761; 0.642734, 0.628515, 0.705077, 0.901193,
0.682813, 0.692832, 0.735423, 0.622034, 0.673723, 0.811797, 0.677923, 0.721698,
0.674035, 0.672388, 0.531461, 0.658120, 0.579842, 0.817870, 0.688333, 1.044625,
0.753991, 0.788593, 0.710741, 0.906241, 0.935628; 0.799106, 0.917052, 1.115237,
1.214851, 1.034818, 1.118828, 1.202928, 0.826798, 1.224967, 1.374303, 1.126406,
0.775190, 0.759131, 1.090059, 0.684655, 0.955639, 0.811285, 1.289494, 1.139661,
0.753991, 1.621034, 1.201765, 1.081841, 1.365902, 1.323655; 0.898689, 0.950677,
1.164796, 1.277434, 1.039625, 1.119519, 1.275592, 1.084260, 1.373886, 1.319088,
1.127440, 0.790003, 0.732206, 1.069719, 0.749979, 0.944118, 0.917944, 1.030990,
0.988895, 0.788593, 1.201765, 1.692948, 1.052744, 1.265542, 1.169388; 0.712825,
0.823301, 0.983688, 1.077374, 0.937088, 0.977618, 1.040266, 0.777897, 1.082813,
1.190385, 1.013104, 0.746242, 0.732658, 0.959136, 0.627718, 0.835790, 0.731996,
1.082950, 0.991869, 0.710741, 1.081841, 1.052744, 1.291439, 1.189060, 1.159221;
0.878405, 1.022355, 1.226847, 1.372354, 1.152792, 1.202464, 1.242863, 0.868127,
1.261720, 1.478896, 1.257194, 0.994816, 0.965557, 1.162012, 0.744096, 1.057347,
0.859613, 1.419900, 1.276105, 0.906241, 1.365902, 1.265542, 1.189060, 1.847657,
1.521522; 0.849565, 0.994935, 1.180170, 1.356724, 1.121475, 1.152186, 1.150213,
0.764337, 1.130003, 1.427420, 1.217591, 1.053857, 1.021876, 1.109763, 0.710310,
1.042521, 0.798469, 1.451896, 1.281761, 0.935628, 1.323655, 1.169388, 1.159221,
1.521522, 1.887527];
q = [-3.640583; -5.563638; -7.040787; -7.387618; -6.410455; -6.452543;
-5.698284; -2.604581; -6.184568; -8.450254; -7.746985; -6.173699; -6.072464;
-5.954310; -2.867163; -5.751653; -3.362224; -8.631145; -8.238493; -4.198840;
-7.653769; -5.764195; -6.387377; -8.917469; -9.348541];
mosekx =
0.0000
0.0000
0.0000
0.0000
0.0000
0.0000
0.0000
0.0000
0.0000
0.0000
1.0951
0.5516
0.6727
0.0000
0.0000
0.0000
0.0000
1.0555
2.0667
0.0000
0.0000
0.0000
0.0000
0.2235
1.1788
QuadraticMinimizer https://github.com/apache/spark/pull/2705
x =
0
0
0
0
0
0
0
0
0
0
1.0951
0.5516
0.6727
0
0
0
0
1.0555
2.0666
0
0
0
0
0.2235
1.1787
I added (step < 1e-7) // too small or negative
> NNLS generates incorrect result
> -------------------------------
>
> Key: SPARK-3987
> URL: https://issues.apache.org/jira/browse/SPARK-3987
> Project: Spark
> Issue Type: Bug
> Components: MLlib
> Affects Versions: 1.1.0
> Reporter: Debasish Das
> Assignee: Shuo Xiang
> Fix For: 1.1.1, 1.2.0
>
>
> Hi,
> Please see the example gram matrix and linear term:
> val P2 = new DoubleMatrix(20, 20, 333907.312770, -60814.043975,
> 207935.829941, -162881.367739, -43730.396770, 17511.428983, -243340.496449,
> -225245.957922, 104700.445881, 32430.845099, 336378.693135, -373497.970207,
> -41147.159621, 53928.060360, -293517.883778, 53105.278068, 0.000000,
> -85257.781696, 84913.970469, -10584.080103, -60814.043975, 13826.806664,
> -38032.612640, 33475.833875, 10791.916809, -1040.950810, 48106.552472,
> 45390.073380, -16310.282190, -2861.455903, -60790.833191, 73109.516544,
> 9826.614644, -8283.992464, 56991.742991, -6171.366034, 0.000000,
> 19152.382499, -13218.721710, 2793.734234, 207935.829941, -38032.612640,
> 129661.677608, -101682.098412, -27401.299347, 10787.713362, -151803.006149,
> -140563.601672, 65067.935324, 20031.263383, 209521.268600, -232958.054688,
> -25764.179034, 33507.951918, -183046.845592, 32884.782835, 0.000000,
> -53315.811196, 52770.762546, -6642.187643, -162881.367739, 33475.833875,
> -101682.098412, 85094.407608, 25422.850782, -5437.646141, 124197.166330,
> 116206.265909, -47093.484134, -11420.168521, -163429.436848, 189574.783900,
> 23447.172314, -24087.375367, 148311.355507, -20848.385466, 0.000000,
> 46835.814559, -38180.352878, 6415.873901, -43730.396770, 10791.916809,
> -27401.299347, 25422.850782, 8882.869799, 15.638084, 35933.473986,
> 34186.371325, -10745.330690, -974.314375, -43537.709621, 54371.010558,
> 7894.453004, -5408.929644, 42231.381747, -3192.010574, 0.000000,
> 15058.753110, -8704.757256, 2316.581535, 17511.428983, -1040.950810,
> 10787.713362, -5437.646141, 15.638084, 2794.949847, -9681.950987,
> -8258.171646, 7754.358930, 4193.359412, 18052.143842, -15456.096769,
> -253.356253, 4089.672804, -12524.380088, 5651.579348, 0.000000, -1513.302547,
> 6296.461898, 152.427321, -243340.496449, 48106.552472, -151803.006149,
> 124197.166330, 35933.473986, -9681.950987, 182931.600236, 170454.352953,
> -72361.174145, -19270.461728, -244518.179729, 279551.060579, 33340.452802,
> -37103.267653, 219025.288975, -33687.141423, 0.000000, 67347.950443,
> -58673.009647, 8957.800259, -225245.957922, 45390.073380, -140563.601672,
> 116206.265909, 34186.371325, -8258.171646, 170454.352953, 159322.942894,
> -66074.960534, -16839.743193, -226173.967766, 260421.044094, 31624.194003,
> -33839.612565, 203889.695169, -30034.828909, 0.000000, 63525.040745,
> -53572.741748, 8575.071847, 104700.445881, -16310.282190, 65067.935324,
> -47093.484134, -10745.330690, 7754.358930, -72361.174145, -66074.960534,
> 35869.598076, 13378.653317, 106033.647837, -111831.682883, -10455.465743,
> 18537.392481, -88370.612394, 20344.288488, 0.000000, -22935.482766,
> 29004.543704, -2409.461759, 32430.845099, -2861.455903, 20031.263383,
> -11420.168521, -974.314375, 4193.359412, -19270.461728, -16839.743193,
> 13378.653317, 6802.081898, 33256.395091, -30421.985199, -1296.785870,
> 7026.518692, -24443.378205, 9221.982599, 0.000000, -4088.076871,
> 10861.014242, -25.092938, 336378.693135, -60790.833191, 209521.268600,
> -163429.436848, -43537.709621, 18052.143842, -244518.179729, -226173.967766,
> 106033.647837, 33256.395091, 339200.268106, -375442.716811, -41027.594509,
> 54636.778527, -295133.248586, 54177.278365, 0.000000, -85237.666701,
> 85996.957056, -10503.209968, -373497.970207, 73109.516544, -232958.054688,
> 189574.783900, 54371.010558, -15456.096769, 279551.060579, 260421.044094,
> -111831.682883, -30421.985199, -375442.716811, 427793.208465, 50528.074431,
> -57375.986301, 335203.382015, -52676.385869, 0.000000, 102368.307670,
> -90679.792485, 13509.390393, -41147.159621, 9826.614644, -25764.179034,
> 23447.172314, 7894.453004, -253.356253, 33340.452802, 31624.194003,
> -10455.465743, -1296.785870, -41027.594509, 50528.074431, 7255.977434,
> -5281.636812, 39298.355527, -3440.450858, 0.000000, 13717.870243,
> -8471.405582, 2071.812204, 53928.060360, -8283.992464, 33507.951918,
> -24087.375367, -5408.929644, 4089.672804, -37103.267653, -33839.612565,
> 18537.392481, 7026.518692, 54636.778527, -57375.986301, -5281.636812,
> 9735.061160, -45360.674033, 10634.633559, 0.000000, -11652.364691,
> 15039.566630, -1202.539106, -293517.883778, 56991.742991, -183046.845592,
> 148311.355507, 42231.381747, -12524.380088, 219025.288975, 203889.695169,
> -88370.612394, -24443.378205, -295133.248586, 335203.382015, 39298.355527,
> -45360.674033, 262923.925938, -42012.606885, 0.000000, 79810.919951,
> -71657.856143, 10464.327491, 53105.278068, -6171.366034, 32884.782835,
> -20848.385466, -3192.010574, 5651.579348, -33687.141423, -30034.828909,
> 20344.288488, 9221.982599, 54177.278365, -52676.385869, -3440.450858,
> 10634.633559, -42012.606885, 13238.686902, 0.000000, -8739.845698,
> 16511.872845, -530.252003, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
> 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
> 0.000000, 0.000000, 0.000000, 0.000000, 118.430000, 0.000000, 0.000000,
> 0.000000, -85257.781696, 19152.382499, -53315.811196, 46835.814559,
> 15058.753110, -1513.302547, 67347.950443, 63525.040745, -22935.482766,
> -4088.076871, -85237.666701, 102368.307670, 13717.870243, -11652.364691,
> 79810.919951, -8739.845698, 0.000000, 26878.133950, -18588.407734,
> 3894.934299, 84913.970469, -13218.721710, 52770.762546, -38180.352878,
> -8704.757256, 6296.461898, -58673.009647, -53572.741748, 29004.543704,
> 10861.014242, 85996.957056, -90679.792485, -8471.405582, 15039.566630,
> -71657.856143, 16511.872845, 0.000000, -18588.407734, 23649.538538,
> -1951.083671, -10584.080103, 2793.734234, -6642.187643, 6415.873901,
> 2316.581535, 152.427321, 8957.800259, 8575.071847, -2409.461759, -25.092938,
> -10503.209968, 13509.390393, 2071.812204, -1202.539106, 10464.327491,
> -530.252003, 0.000000, 3894.934299, -1951.083671, 738.955915)
> val q2 = new DoubleMatrix(20, 1, 31755.057100, -13047.148129,
> 20191.244430, -25993.775800, -11963.550172, -4272.425977, -33569.856044,
> -33451.387021, 2320.764250, -5333.136834, 30633.764272, -49513.939049,
> -10351.230305, 872.276714, -37634.078430, -4628.338543, -0.000000,
> -18109.093788, 1856.725521, -3397.693211)
> NNLS result:
> 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
> PR result: https://github.com/apache/spark/pull/2705
> QuadraticMinimizer result [0.130104; 0.126840; 0.072565; 0.154153; 0.144728;
> 0.129937; 0.121166; 0.161450; 0.199197; 0.187168; 0.159467; 0.144269;
> 0.117404; 0.109298; 0.086901; 0.221391; 0.000000; 0.174044; 0.162080;
> 0.045439]
> Octave result:
> octave:7> qp(x0, H, f, [], [], lb, ub)
> ans =
> 0.13010
> 0.12684
> 0.07256
> 0.15415
> 0.14473
> 0.12994
> 0.12117
> 0.16145
> 0.19920
> 0.18717
> 0.15947
> 0.14427
> 0.11740
> 0.10930
> 0.08690
> 0.22139
> 0.00000
> 0.17404
> 0.16208
> 0.04544
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
