kaknikhil commented on a change in pull request #425: DL: Add training for
multiple models
URL: https://github.com/apache/madlib/pull/425#discussion_r310245940
##########
File path: src/ports/postgres/modules/deep_learning/madlib_keras.py_in
##########
@@ -403,49 +408,84 @@ def should_compute_metrics_this_iter(curr_iter,
metrics_compute_frequency,
return (curr_iter)%metrics_compute_frequency == 0 or \
curr_iter == num_iterations
+
def fit_transition(state, dependent_var, independent_var, model_architecture,
compile_params, fit_params, current_seg_id, seg_ids,
images_per_seg, gpus_per_host, segments_per_host,
- prev_serialized_weights, **kwargs):
+ prev_serialized_weights, is_final=True,
+ is_cerebro=False, **kwargs):
+
if not independent_var or not dependent_var:
return state
start_transition = time.time()
SD = kwargs['SD']
device_name = get_device_name_and_set_cuda_env(gpus_per_host,
current_seg_id)
- # Set up system if this is the first buffer on segment'
- if not state:
- set_keras_session(device_name, gpus_per_host, segments_per_host)
- segment_model = model_from_json(model_architecture)
- compile_and_set_weights(segment_model, compile_params, device_name,
- prev_serialized_weights)
- SD['segment_model'] = segment_model
- agg_image_count = 0
+ # Cerebro state has the weights passed from the previous segment
+ if is_cerebro:
+ prev_serialized_weights = madlib_keras_serializer.\
+ get_serialized_1d_weights_from_state(prev_serialized_weights)
+
+ # If a live session is present, re-use it. Otherwise, recreate it.
+ if 'sess' in SD:
+ sess = SD['sess']
else:
- segment_model = SD['segment_model']
- agg_image_count =
madlib_keras_serializer.get_image_count_from_state(state)
+ sess = get_keras_session(device_name, gpus_per_host, segments_per_host)
+ SD['sess'] = sess
+ K.set_session(sess)
+
+ # Set up system if this is the first buffer on segment.
+
+ # On cerebro, clean the session on the first row.
+ if is_cerebro:
+ if not state:
+ K.clear_session()
+ segment_model = model_from_json(model_architecture)
+ compile_and_set_weights(segment_model, compile_params, device_name,
+ prev_serialized_weights)
+ SD['segment_model'] = segment_model
+ agg_image_count = 0
+ else:
+ all_ops_len = len([n.name for n in
tf.get_default_graph().as_graph_def().node])
+ segment_model = SD['segment_model']
+ agg_image_count =
madlib_keras_serializer.get_image_count_from_state(state)
+
+ # On single model fit, reuse the model if it exists
+ else:
+ if not state:
+ agg_image_count = 0
+ else:
+ agg_image_count =
madlib_keras_serializer.get_image_count_from_state(state)
+ if 'segment_model' in SD:
+ segment_model = SD['segment_model']
+ if not state:
Review comment:
state should never be null here since segment_model is only set once for the
first row when the state is null. Unless I am missing something, I think we can
delete lines 463 - 467
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
