fmcquillan99 commented on issue #443: DL: Add training for multiple models URL: https://github.com/apache/madlib/pull/443#issuecomment-540664010 Prelim testing for fit() looks OK (see below). I suggest after @kaknikhil finishes addressing review comments that we merge this PR then work on https://issues.apache.org/jira/browse/MADLIB-1387 after which we can do more E2E testing. ``` SELECT * FROM mst_table ORDER BY mst_key; mst_key | model_id | compile_params | fit_params ---------+----------+---------------------------------------------------------------------------------+----------------------- 1 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 2 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 3 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 4 | 1 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 5 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 6 | 1 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 7 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=4,epochs=1 8 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] | batch_size=8,epochs=1 9 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=4,epochs=1 10 | 2 | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] | batch_size=8,epochs=1 11 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=4,epochs=1 12 | 2 | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] | batch_size=8,epochs=1 (12 rows) DROP TABLE IF EXISTS iris_multi_model, iris_multi_model_summary, iris_multi_model_info; SELECT madlib.madlib_keras_fit_multiple_model('iris_train_packed', -- source_table 'iris_multi_model', -- model_output_table 'mst_table', -- model_selection_table 7, -- num_iterations 0, -- gpus_per_host 'iris_test_packed' -- validation dataset ); SELECT * FROM iris_multi_model_info ORDER BY training_metrics_final DESC; -[ RECORD 1 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 10 model_id | 2 compile_params | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {0.181155920028687,0.186516046524048,0.184003829956055,0.258072137832642,0.192763805389404,0.18028712272644,0.176060914993286} metrics_type | {accuracy} training_metrics_final | 0.983333349228 training_loss_final | 0.249839022756 training_metrics | {0.716666638851166,0.716666638851166,0.716666638851166,0.716666638851166,0.716666638851166,0.783333361148834,0.983333349227905} training_loss | {0.828127026557922,0.647694051265717,0.481598526239395,0.439306199550629,0.393612682819366,0.308133810758591,0.249839022755623} validation_metrics_final | 0.899999976158 validation_loss_final | 0.335877805948 validation_metrics | {0.466666668653488,0.466666668653488,0.466666668653488,0.466666668653488,0.5,0.633333325386047,0.899999976158142} validation_loss | {1.13481867313385,0.979182302951813,0.748289108276367,0.746902406215668,0.66666042804718,0.500535547733307,0.335877805948257} -[ RECORD 2 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 8 model_id | 2 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {0.193707942962646,0.177205085754395,0.188807010650635,0.177771091461182,0.188723087310791,0.178416967391968,0.199656009674072} metrics_type | {accuracy} training_metrics_final | 0.983333349228 training_loss_final | 0.159384667873 training_metrics | {0.716666638851166,0.708333313465118,0.699999988079071,0.716666638851166,0.725000023841858,0.550000011920929,0.983333349227905} training_loss | {0.623141229152679,0.618586182594299,0.520916938781738,0.485385835170746,0.456164866685867,0.781868517398834,0.159384667873383} validation_metrics_final | 0.866666674614 validation_loss_final | 0.247822448611 validation_metrics | {0.466666668653488,0.466666668653488,0.466666668653488,0.466666668653488,0.5,0.200000002980232,0.866666674613953} validation_loss | {0.630768358707428,0.811316192150116,0.619273781776428,0.66645485162735,0.520728230476379,0.992052137851715,0.247822448611259} -[ RECORD 3 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 4 model_id | 1 compile_params | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {0.15691614151001,0.153699159622192,0.548386812210083,0.169425010681152,0.165678024291992,0.157116889953613,0.15025782585144} metrics_type | {accuracy} training_metrics_final | 0.975000023842 training_loss_final | 0.26616576314 training_metrics | {0.391666680574417,0.400000005960464,0.725000023841858,0.925000011920929,0.758333325386047,0.833333313465118,0.975000023841858} training_loss | {1.02847397327423,0.830727696418762,0.655663847923279,0.534718573093414,0.419764846563339,0.321459263563156,0.266165763139725} validation_metrics_final | 0.933333337307 validation_loss_final | 0.310080826283 validation_metrics | {0.100000001490116,0.100000001490116,0.5,0.833333313465118,0.533333361148834,0.733333349227905,0.933333337306976} validation_loss | {1.24580788612366,1.13383376598358,0.899369597434998,0.65422135591507,0.657262623310089,0.500903606414795,0.310080826282501} -[ RECORD 4 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 7 model_id | 2 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] fit_params | batch_size=4,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {0.192800045013428,0.176566123962402,0.184437036514282,0.179429054260254,0.182589054107666,0.176466941833496,0.202085018157959} metrics_type | {accuracy} training_metrics_final | 0.925000011921 training_loss_final | 0.321873039007 training_metrics | {0.608333349227905,0.975000023841858,0.75,0.858333349227905,0.608333349227905,0.983333349227905,0.925000011920929} training_loss | {0.649360001087189,0.362144291400909,0.394706398248672,0.236399039626122,0.553112983703613,0.327443867921829,0.321873039007187} validation_metrics_final | 1 validation_loss_final | 0.241945460439 validation_metrics | {0.899999976158142,0.866666674613953,0.533333361148834,0.733333349227905,0.899999976158142,0.866666674613953,1} validation_loss | {0.332620888948441,0.502064645290375,0.66054779291153,0.538120031356812,0.234018862247467,0.337950766086578,0.241945460438728} -[ RECORD 5 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 3 model_id | 1 compile_params | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] fit_params | batch_size=4,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {0.15620493888855,0.155749082565308,0.193189859390259,0.166114091873169,0.156448841094971,0.148685932159424,0.152026891708374} metrics_type | {accuracy} training_metrics_final | 0.925000011921 training_loss_final | 0.325599312782 training_metrics | {0.391666680574417,0.725000023841858,0.725000023841858,0.716666638851166,0.75,0.716666638851166,0.925000011920929} training_loss | {1.11191356182098,0.621958792209625,0.446929544210434,0.41201776266098,0.374244391918182,0.509410500526428,0.325599312782288} validation_metrics_final | 0.933333337307 validation_loss_final | 0.276731878519 validation_metrics | {0.100000001490116,0.5,0.466666668653488,0.466666668653488,0.533333361148834,0.466666668653488,0.933333337306976} validation_loss | {1.63468599319458,0.65976870059967,0.540513038635254,0.567835509777069,0.486706107854843,0.917201280593872,0.276731878519058} -[ RECORD 6 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 9 model_id | 2 compile_params | loss='categorical_crossentropy', optimizer='Adam(lr=0.01)',metrics=['accuracy'] fit_params | batch_size=4,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {0.546731948852539,0.173670053482056,0.189447164535522,0.18386697769165,0.179111003875732,0.179292917251587,0.17428994178772} metrics_type | {accuracy} training_metrics_final | 0.891666650772 training_loss_final | 0.208736971021 training_metrics | {0.708333313465118,0.866666674613953,0.758333325386047,0.933333337306976,0.991666674613953,0.983333349227905,0.891666650772095} training_loss | {0.753945171833038,0.351552069187164,0.330372422933578,0.267056792974472,0.148756921291351,0.110077999532223,0.208736971020699} validation_metrics_final | 1 validation_loss_final | 0.0952743515372 validation_metrics | {0.466666668653488,0.699999988079071,0.566666662693024,0.899999976158142,0.899999976158142,0.933333337306976,1} validation_loss | {1.06321632862091,0.520808517932892,0.602703869342804,0.26708459854126,0.246414661407471,0.179920583963394,0.0952743515372276} -[ RECORD 7 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 11 model_id | 2 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] fit_params | batch_size=4,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {0.186348915100098,0.186259984970093,0.176491975784302,0.574105024337769,0.191556930541992,0.178547859191895,0.176308870315552} metrics_type | {accuracy} training_metrics_final | 0.716666638851 training_loss_final | 0.478766709566 training_metrics | {0.391666680574417,0.591666638851166,0.716666638851166,0.716666638851166,0.716666638851166,0.716666638851166,0.716666638851166} training_loss | {0.952429413795471,0.849565386772156,0.773178815841675,0.676037073135376,0.601148724555969,0.532010197639465,0.478766709566116} validation_metrics_final | 0.466666668653 validation_loss_final | 0.658966064453 validation_metrics | {0.100000001490116,0.366666674613953,0.466666668653488,0.466666668653488,0.466666668653488,0.466666668653488,0.466666668653488} validation_loss | {1.31364703178406,1.14900648593903,1.0735125541687,0.92445170879364,0.823235094547272,0.713083922863007,0.658966064453125} -[ RECORD 8 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 6 model_id | 1 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {0.158112049102783,0.153064966201782,0.175857067108154,0.151686906814575,0.162416934967041,0.590201854705811,0.161259889602661} metrics_type | {accuracy} training_metrics_final | 0.716666638851 training_loss_final | 0.896975934505 training_metrics | {0.391666680574417,0.658333361148834,0.716666638851166,0.716666638851166,0.716666638851166,0.716666638851166,0.716666638851166} training_loss | {1.08892619609833,1.05738139152527,1.02508842945099,0.996677756309509,0.966224551200867,0.930180728435516,0.896975934505463} validation_metrics_final | 0.466666668653 validation_loss_final | 1.07026696205 validation_metrics | {0.100000001490116,0.400000005960464,0.466666668653488,0.466666668653488,0.466666668653488,0.466666668653488,0.466666668653488} validation_loss | {1.36965489387512,1.33375680446625,1.25807249546051,1.22709584236145,1.17450451850891,1.09602510929108,1.07026696205139} -[ RECORD 9 ]------------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 12 model_id | 2 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 1.2197265625 metrics_elapsed_time | {0.182577848434448,0.180651903152466,0.191632986068726,0.184897899627686,0.179800987243652,0.175981998443604,0.174694776535034} metrics_type | {accuracy} training_metrics_final | 0.708333313465 training_loss_final | 0.879215478897 training_metrics | {0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.400000005960464,0.558333337306976,0.708333313465118} training_loss | {2.4544358253479,1.88577473163605,1.46488261222839,1.21406507492065,1.06772172451019,0.960707426071167,0.879215478897095} validation_metrics_final | 0.466666668653 validation_loss_final | 1.12123858929 validation_metrics | {0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.400000005960464,0.466666668653488} validation_loss | {3.79992151260376,2.91293883323669,2.22414708137512,1.76915240287781,1.4565817117691,1.25908029079437,1.1212385892868} -[ RECORD 10 ]-----------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 2 model_id | 1 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] fit_params | batch_size=8,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {0.159732103347778,0.156741142272949,0.152164936065674,0.168025970458984,0.166567087173462,0.153841018676758,0.148435115814209} metrics_type | {accuracy} training_metrics_final | 0.391666680574 training_loss_final | 1.09972310066 training_metrics | {0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417} training_loss | {1.09512889385223,1.09819459915161,1.09368872642517,1.0922759771347,1.10061180591583,1.09582090377808,1.09972310066223} validation_metrics_final | 0.10000000149 validation_loss_final | 1.27448761463 validation_metrics | {0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116} validation_loss | {1.20508420467377,1.26680672168732,1.24172151088715,1.205646276474,1.27797186374664,1.25544846057892,1.27448761463165} -[ RECORD 11 ]-----------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 1 model_id | 1 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.1)',metrics=['accuracy'] fit_params | batch_size=4,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {0.168879985809326,0.152376174926758,0.156481981277466,0.152595996856689,0.207169055938721,0.156559944152832,0.654519081115723} metrics_type | {accuracy} training_metrics_final | 0.391666680574 training_loss_final | 1.11036765575 training_metrics | {0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417} training_loss | {1.10677635669708,1.09215080738068,1.11154639720917,1.10222017765045,1.09642863273621,1.09669518470764,1.11036765575409} validation_metrics_final | 0.10000000149 validation_loss_final | 1.31826531887 validation_metrics | {0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116} validation_loss | {1.31928586959839,1.19879269599915,1.3339284658432,1.29005479812622,1.26196086406708,1.2497181892395,1.31826531887054} -[ RECORD 12 ]-----------+-------------------------------------------------------------------------------------------------------------------------------- mst_key | 5 model_id | 1 compile_params | loss='categorical_crossentropy',optimizer='Adam(lr=0.001)',metrics=['accuracy'] fit_params | batch_size=4,epochs=1 model_type | madlib_keras model_size | 0.7900390625 metrics_elapsed_time | {0.163282155990601,0.152222871780396,0.55925989151001,0.153285980224609,0.155977010726929,0.159151077270508,0.169466972351074} metrics_type | {accuracy} training_metrics_final | 0.391666680574 training_loss_final | 0.94362616539 training_metrics | {0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.391666680574417,0.400000005960464,0.391666680574417} training_loss | {1.31070482730865,1.1811283826828,1.1090475320816,1.05325126647949,1.01403939723969,0.977032721042633,0.943626165390015} validation_metrics_final | 0.10000000149 validation_loss_final | 1.20139348507 validation_metrics | {0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116,0.100000001490116} validation_loss | {1.90102434158325,1.61195623874664,1.46869885921478,1.3178962469101,1.27622199058533,1.26882350444794,1.20139348506927} ```
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
