makemebitter commented on a change in pull request #425: DL: Add training for
multiple models
URL: https://github.com/apache/madlib/pull/425#discussion_r310785453
##########
File path: src/ports/postgres/modules/deep_learning/madlib_keras.py_in
##########
@@ -403,49 +408,84 @@ def should_compute_metrics_this_iter(curr_iter,
metrics_compute_frequency,
return (curr_iter)%metrics_compute_frequency == 0 or \
curr_iter == num_iterations
+
def fit_transition(state, dependent_var, independent_var, model_architecture,
compile_params, fit_params, current_seg_id, seg_ids,
images_per_seg, gpus_per_host, segments_per_host,
- prev_serialized_weights, **kwargs):
+ prev_serialized_weights, is_final=True,
+ is_cerebro=False, **kwargs):
+
if not independent_var or not dependent_var:
return state
start_transition = time.time()
SD = kwargs['SD']
device_name = get_device_name_and_set_cuda_env(gpus_per_host,
current_seg_id)
- # Set up system if this is the first buffer on segment'
- if not state:
- set_keras_session(device_name, gpus_per_host, segments_per_host)
- segment_model = model_from_json(model_architecture)
- compile_and_set_weights(segment_model, compile_params, device_name,
- prev_serialized_weights)
- SD['segment_model'] = segment_model
- agg_image_count = 0
+ # Cerebro state has the weights passed from the previous segment
+ if is_cerebro:
+ prev_serialized_weights = madlib_keras_serializer.\
+ get_serialized_1d_weights_from_state(prev_serialized_weights)
+
+ # If a live session is present, re-use it. Otherwise, recreate it.
+ if 'sess' in SD:
+ sess = SD['sess']
else:
- segment_model = SD['segment_model']
- agg_image_count =
madlib_keras_serializer.get_image_count_from_state(state)
+ sess = get_keras_session(device_name, gpus_per_host, segments_per_host)
+ SD['sess'] = sess
+ K.set_session(sess)
+
+ # Set up system if this is the first buffer on segment.
+
+ # On cerebro, clean the session on the first row.
+ if is_cerebro:
+ if not state:
+ K.clear_session()
+ segment_model = model_from_json(model_architecture)
+ compile_and_set_weights(segment_model, compile_params, device_name,
+ prev_serialized_weights)
+ SD['segment_model'] = segment_model
+ agg_image_count = 0
+ else:
+ all_ops_len = len([n.name for n in
tf.get_default_graph().as_graph_def().node])
+ segment_model = SD['segment_model']
+ agg_image_count =
madlib_keras_serializer.get_image_count_from_state(state)
+
+ # On single model fit, reuse the model if it exists
+ else:
+ if not state:
+ agg_image_count = 0
+ else:
+ agg_image_count =
madlib_keras_serializer.get_image_count_from_state(state)
+ if 'segment_model' in SD:
+ segment_model = SD['segment_model']
+ if not state:
Review comment:
state can be null if it's the first row of a new iteration, and the
segment_model is being reused from the previous iteration.
segment_model is initialized only at the first row of the first iteration
and kept in SD for reuse in all subsequent iterations.
The code here is to update the weights of segment_model to the ones passed
in from the UDA, which is the globally averaged weights from the previous
iteration. We can't delete these as the segment_model at this stage, has only
the local weights.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
