Repository: reef Updated Branches: refs/heads/master b1d7a7013 -> 22c047754
[REEF-1229] Make sure Evaluator PID is written for short lived Evaluators This makes `PIDStoreStartHandler` an `EventHandler<RuntimeStart>` and adds synchronization logic in `EvaluatorRuntime` to make sure it is always invoked. Also, this removes the double bindings of `PIDStoreStartHandler`, which was bound in the various runtimes as well as in `REEFLauncher`. This change binds it only in `REEFLauncher` JIRA: [REEF-1229](https://issues.apache.org/jira/browse/REEF-1229) Pull Request: This closes #866 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/22c04775 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/22c04775 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/22c04775 Branch: refs/heads/master Commit: 22c0477545557297b35f812116e21aa273cd0d12 Parents: b1d7a70 Author: Markus Weimer <[email protected]> Authored: Mon Feb 29 10:22:02 2016 -0800 Committer: Boris Shulman <[email protected]> Committed: Mon Feb 29 15:57:12 2016 -0800 ---------------------------------------------------------------------- .../reef/runtime/common/REEFLauncher.java | 2 +- .../common/evaluator/EvaluatorRuntime.java | 5 ++ .../common/evaluator/PIDStoreStartHandler.java | 57 +++++++++++++++----- .../ExtensibleLocalRuntimeConfiguration.java | 3 -- .../local/client/LocalRuntimeConfiguration.java | 3 -- .../client/StandaloneRuntimeConfiguration.java | 5 -- 6 files changed, 50 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/22c04775/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/REEFLauncher.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/REEFLauncher.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/REEFLauncher.java index 9356f55..ad43373 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/REEFLauncher.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/REEFLauncher.java @@ -62,7 +62,7 @@ public final class REEFLauncher { private static final Logger LOG = Logger.getLogger(REEFLauncher.class.getName()); private static final Configuration LAUNCHER_STATIC_CONFIG = Tang.Factory.getTang().newConfigurationBuilder() - .bindSetEntry(Clock.StartHandler.class, PIDStoreStartHandler.class) + .bindSetEntry(Clock.RuntimeStartHandler.class, PIDStoreStartHandler.class) .bindNamedParameter(RemoteConfiguration.ErrorHandler.class, REEFErrorHandler.class) .bindNamedParameter(RemoteConfiguration.ManagerName.class, "REEF_LAUNCHER") .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class) http://git-wip-us.apache.org/repos/asf/reef/blob/22c04775/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/EvaluatorRuntime.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/EvaluatorRuntime.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/EvaluatorRuntime.java index 0584f58..729c2e2 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/EvaluatorRuntime.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/EvaluatorRuntime.java @@ -49,6 +49,7 @@ final class EvaluatorRuntime implements EventHandler<EvaluatorControlProto> { private final HeartBeatManager heartBeatManager; private final ContextManager contextManager; private final Clock clock; + private final PIDStoreStartHandler pidStoreStartHandler; private final String evaluatorIdentifier; private final ExceptionCodec exceptionCodec; @@ -66,6 +67,7 @@ final class EvaluatorRuntime implements EventHandler<EvaluatorControlProto> { final Clock clock, final ContextManager contextManagerFuture, final RemoteManager remoteManager, + final PIDStoreStartHandler pidStoreStartHandler, final ExceptionCodec exceptionCodec) { this.heartBeatManager = heartBeatManager; @@ -73,6 +75,7 @@ final class EvaluatorRuntime implements EventHandler<EvaluatorControlProto> { this.clock = clock; this.evaluatorIdentifier = evaluatorIdentifier; + this.pidStoreStartHandler = pidStoreStartHandler; this.exceptionCodec = exceptionCodec; this.evaluatorControlChannel = remoteManager.registerHandler(driverRID, EvaluatorControlProto.class, this); @@ -169,6 +172,8 @@ final class EvaluatorRuntime implements EventHandler<EvaluatorControlProto> { @Override @SuppressWarnings("checkstyle:illegalcatch") public void onNext(final RuntimeStart runtimeStart) { + // [REEF-1229] Make sure that the PID is always written before we potentially exit the Evaluator process. + EvaluatorRuntime.this.pidStoreStartHandler.onNext(runtimeStart); synchronized (EvaluatorRuntime.this.heartBeatManager) { try { LOG.log(Level.FINEST, "runtime start"); http://git-wip-us.apache.org/repos/asf/reef/blob/22c04775/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/PIDStoreStartHandler.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/PIDStoreStartHandler.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/PIDStoreStartHandler.java index 3f5536f..03091a9 100644 --- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/PIDStoreStartHandler.java +++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/PIDStoreStartHandler.java @@ -20,34 +20,65 @@ package org.apache.reef.runtime.common.evaluator; import org.apache.reef.util.OSUtils; import org.apache.reef.wake.EventHandler; -import org.apache.reef.wake.time.event.StartTime; +import org.apache.reef.wake.time.runtime.event.RuntimeStart; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; import javax.inject.Inject; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; import java.util.logging.Level; import java.util.logging.Logger; /** * This Handler writes the Process ID (PID) to a file with a name given in PID_FILE_NAME to the local working directory. */ -public class PIDStoreStartHandler implements EventHandler<StartTime> { +@ThreadSafe +public final class PIDStoreStartHandler implements EventHandler<RuntimeStart> { + /** + * The file name of the PID file created in the current working directory of the process. + */ public static final String PID_FILE_NAME = "PID.txt"; + private static final Logger LOG = Logger.getLogger(PIDStoreStartHandler.class.getName()); + @GuardedBy("this") + private boolean pidIsWritten = false; + @Inject - public PIDStoreStartHandler() { + private PIDStoreStartHandler() { } + /** + * This call is idempotent: It will only write the PID exactly once per instance. + * + * @param startTime + */ @Override - public void onNext(final StartTime startTime) { - final long pid = OSUtils.getPID(); - final File outfile = new File(PID_FILE_NAME); - LOG.log(Level.FINEST, "Storing pid `" + pid + "` in file " + outfile.getAbsolutePath()); - try (final PrintWriter p = new PrintWriter(PID_FILE_NAME, "UTF-8")) { - p.write(String.valueOf(pid)); - p.write("\n"); - } catch (final FileNotFoundException | UnsupportedEncodingException e) { - LOG.log(Level.WARNING, "Unable to create PID file.", e); + public synchronized void onNext(final RuntimeStart startTime) { + if (this.isPidNotWritten()) { + final long pid = OSUtils.getPID(); + final File outfile = new File(PID_FILE_NAME); + LOG.log(Level.FINEST, "Storing pid `" + pid + "` in file " + outfile.getAbsolutePath()); + try (final PrintWriter p = new PrintWriter(PID_FILE_NAME, "UTF-8")) { + p.write(String.valueOf(pid)); + p.write("\n"); + } catch (final FileNotFoundException | UnsupportedEncodingException e) { + LOG.log(Level.WARNING, "Unable to create PID file.", e); + } + this.pidIsWritten = true; + } else { + LOG.log(Level.FINEST, "PID already written."); } } + + + /** + * @return true, if the PID hasn't been written yet. + */ + private synchronized boolean isPidNotWritten() { + return !this.pidIsWritten; + } } http://git-wip-us.apache.org/repos/asf/reef/blob/22c04775/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/ExtensibleLocalRuntimeConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/ExtensibleLocalRuntimeConfiguration.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/ExtensibleLocalRuntimeConfiguration.java index c8a3f45..733dc69 100644 --- a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/ExtensibleLocalRuntimeConfiguration.java +++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/ExtensibleLocalRuntimeConfiguration.java @@ -22,7 +22,6 @@ import org.apache.reef.client.parameters.DriverConfigurationProviders; import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration; import org.apache.reef.runtime.common.client.DriverConfigurationProvider; import org.apache.reef.runtime.common.client.api.JobSubmissionHandler; -import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler; import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; import org.apache.reef.runtime.common.parameters.JVMHeapSlack; import org.apache.reef.runtime.local.LocalClasspathProvider; @@ -31,7 +30,6 @@ import org.apache.reef.runtime.local.client.parameters.RackNames; import org.apache.reef.runtime.local.client.parameters.RootFolder; import org.apache.reef.tang.ConfigurationProvider; import org.apache.reef.tang.formats.*; -import org.apache.reef.wake.time.Clock; import java.util.concurrent.ExecutorService; @@ -91,7 +89,6 @@ public final class ExtensibleLocalRuntimeConfiguration extends ConfigurationModu .bindNamedParameter(RootFolder.class, RUNTIME_ROOT_FOLDER) .bindNamedParameter(JVMHeapSlack.class, JVM_HEAP_SLACK) .bindSetEntry(DriverConfigurationProviders.class, DRIVER_CONFIGURATION_PROVIDERS) - .bindSetEntry(Clock.StartHandler.class, PIDStoreStartHandler.class) .bindSetEntry(RackNames.class, RACK_NAMES) .build(); } http://git-wip-us.apache.org/repos/asf/reef/blob/22c04775/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java index c8282b9..3236680 100644 --- a/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java +++ b/lang/java/reef-runtime-local/src/main/java/org/apache/reef/runtime/local/client/LocalRuntimeConfiguration.java @@ -22,7 +22,6 @@ import org.apache.reef.client.parameters.DriverConfigurationProviders; import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration; import org.apache.reef.runtime.common.client.DriverConfigurationProvider; import org.apache.reef.runtime.common.client.api.JobSubmissionHandler; -import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler; import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; import org.apache.reef.runtime.common.parameters.JVMHeapSlack; import org.apache.reef.runtime.local.LocalClasspathProvider; @@ -31,7 +30,6 @@ import org.apache.reef.runtime.local.client.parameters.RackNames; import org.apache.reef.runtime.local.client.parameters.RootFolder; import org.apache.reef.tang.ConfigurationProvider; import org.apache.reef.tang.formats.*; -import org.apache.reef.wake.time.Clock; import java.util.concurrent.ExecutorService; @@ -86,7 +84,6 @@ public class LocalRuntimeConfiguration extends ConfigurationModuleBuilder { .bindNamedParameter(RootFolder.class, RUNTIME_ROOT_FOLDER) .bindNamedParameter(JVMHeapSlack.class, JVM_HEAP_SLACK) .bindSetEntry(DriverConfigurationProviders.class, DRIVER_CONFIGURATION_PROVIDERS) - .bindSetEntry(Clock.StartHandler.class, PIDStoreStartHandler.class) .bindSetEntry(RackNames.class, RACK_NAMES) .build(); http://git-wip-us.apache.org/repos/asf/reef/blob/22c04775/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java index cb3ca3a..5c64539 100644 --- a/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java +++ b/lang/java/reef-runtime-standalone/src/main/java/org/apache/reef/runtime/standalone/client/StandaloneRuntimeConfiguration.java @@ -21,12 +21,10 @@ package org.apache.reef.runtime.standalone.client; import org.apache.reef.annotations.Unstable; import org.apache.reef.client.parameters.DriverConfigurationProviders; import org.apache.reef.runtime.common.client.CommonRuntimeConfiguration; -import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler; import org.apache.reef.runtime.standalone.client.parameters.NodeListFilePath; import org.apache.reef.runtime.standalone.client.parameters.RootFolder; import org.apache.reef.tang.ConfigurationProvider; import org.apache.reef.tang.formats.*; -import org.apache.reef.wake.time.Clock; /** * A ConfigurationModule to configure the standalone resourcemanager. @@ -62,8 +60,5 @@ public final class StandaloneRuntimeConfiguration extends ConfigurationModuleBui .bindNamedParameter(RootFolder.class, RUNTIME_ROOT_FOLDER) .bindNamedParameter(NodeListFilePath.class, NODE_LIST_FILE_PATH) .bindSetEntry(DriverConfigurationProviders.class, DRIVER_CONFIGURATION_PROVIDERS) - .bindSetEntry(Clock.StartHandler.class, PIDStoreStartHandler.class) .build(); - - }
