[
https://issues.apache.org/jira/browse/MADLIB-1456?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17266482#comment-17266482
]
Frank McQuillan commented on MADLIB-1456:
-----------------------------------------
see results in https://github.com/apache/madlib/pull/528
> Hyperband cumulative output time is not correct
> -----------------------------------------------
>
> Key: MADLIB-1456
> URL: https://issues.apache.org/jira/browse/MADLIB-1456
> Project: Apache MADlib
> Issue Type: Bug
> Components: Deep Learning
> Reporter: Frank McQuillan
> Priority: Minor
> Fix For: v1.18.0
>
>
> {code}
> SELECT madlib.madlib_keras_automl('cifar10_train_packed',
> 'automl_cifar10_output',
> 'model_arch_library',
> 'automl_cifar10_mst_table',
> ARRAY[1,2,3],
> $${'loss': ['categorical_crossentropy'],
> 'optimizer_params_list': [
> {'optimizer': ['Adam'],'lr':
> [0.0001, 0.01, 'log']},
> {'optimizer': ['RMSprop'],'lr':
> [0.0001, 0.01, 'log'], 'decay': [1.0e-7, 1.0e-5, 'log']},
> {'optimizer': ['SGD'],'lr': [0.0001,
> 0.01, 'log'], 'momentum': [0.9, 0.99,'log_near_one']}],
> 'metrics':['accuracy']}$$,
> $${'batch_size': [64,128], 'epochs': [1]}$$,
> 'hyperband',
> 'R=27, eta=3, skip_last=1',
> NULL, -- random state
> NULL, -- object table
> FALSE, -- use GPUs
> 'cifar10_val_packed', -- validation table
> 3, -- metrics compute
> freq
> NULL, -- name
> NULL); -- descr
> {code}
> produces
> {code}
> select mst_key, metrics_iters, metrics_elapsed_time from
> hb_cifar10_output_info_r27e3skip1run1 order by mst_key, metrics_iters;
> mst_key | metrics_iters |
> metrics_elapsed_time
> ---------+---------------+----------------------------------------------------------------------------------------
> 1 | {1} | {5639.45646381378}
> 2 | {1,4} | {5499.1161878109,10417.6853170395}
> 3 | {1} | {5647.45680689812}
> 4 | {1,4} | {5581.32227087021,10422.9603590965}
> 5 | {1} | {5473.09724283218}
> 6 | {1} | {5537.51781392097}
> 7 | {1} | {5468.15699982643}
> 8 | {1} | {5597.43572378159}
> 9 | {1,4} | {5618.64315199852,10291.4588999748}
> 10 | {1} | {5450.7096657753}
> 11 | {1,4,7,10,13} |
> {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
> 12 | {1} | {5674.29211378098}
> 13 | {1} | {5542.49673891068}
> 14 | {1} | {5682.2088098526}
> 15 | {1,4,7,10,13} |
> {5610.54115080833,10365.9494991302,7378.03450798988,14876.2571978569,22414.0461809635}
> 16 | {1} | {5563.48544478416}
> 17 | {1,4,7,10,13} |
> {5661.10293698311,10398.6006779671,7373.00771188736,14871.3066940308,22409.0431649685}
> 18 | {1} | {5626.78051185608}
> 19 | {1} | {5656.17028284073}
> 20 | {1} | {5524.58638978004}
> 21 | {1} | {5555.59004878998}
> 22 | {1} | {5605.62219786644}
> 23 | {1} | {5494.16033291817}
> 24 | {1,4} | {5512.02432179451,10360.8255741596}
> 25 | {1,4} | {5455.62597084045,10296.5835280418}
> 26 | {1,4} | {5589.49808478355,10337.8003959656}
> 27 | {1} | {5576.40678977966}
> 28 | {4,7,10,13} |
> {10393.5381979942,7413.2452878952,14912.529984951,22449.6516678333}
> 29 | {4} | {10319.5200190544}
> 30 | {4} | {10451.2104020119}
> 31 | {4,7,10,13} |
> {10328.5663330555,7399.8391699791,14899.122522831,22436.2282240391}
> 32 | {4} | {10437.2220571041}
> 33 | {4} | {10379.5259420872}
> 34 | {4} | {10346.9251220226}
> 35 | {4,7,10,13} |
> {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
> 36 | {4} | {10412.5241000652}
> 37 | {7,10,13} | {7421.8063390255,14921.1340680122,22459.3360388279}
> 38 | {7,10,13} | {7469.96136283875,14970.6747159958,22509.476047039}
> 39 | {7,10,13} |
> {7461.39016890526,14962.0603349209,22500.7105178833}
> 40 | {7,10,13} | {7435.0908639431,14935.0127699375,22472.9902210236}
> 41 | {7,10,13} |
> {7391.56045103073,14890.2629489899,22427.4269089699}
> 42 | {7,10,13} | {7354.62691497803,14852.9146559238,22389.92344594}
> (42 rows)
> {code}
> Note that for times with 7 cumulative time is wrong, here are 2 exs:
> {code}
> 11 | {1,4,7,10,13} |
> {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
> and
> 35 | {4,7,10,13} |
> {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
> {code}
> The expected schedule is:
> {code}
> DROP TABLE IF EXISTS hb_schedule;
> SELECT madlib.hyperband_schedule ('hb_schedule',
> 27,
> 3,
> 1);
> SELECT * FROM hb_schedule ORDER BY s DESC, i;
> s | i | n_i | r_i
> ---+---+-----+-----
> 3 | 0 | 27 | 1
> 3 | 1 | 9 | 3
> 3 | 2 | 3 | 9
> 2 | 0 | 9 | 3
> 2 | 1 | 3 | 9
> 1 | 0 | 6 | 9
> (6 rows)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
