Author: burn
Date: Fri Nov 4 14:33:37 2016
New Revision: 1768041
URL: http://svn.apache.org/viewvc?rev=1768041&view=rev
Log:
UIMA-5170 Skip restore of work that shares an already-processed experiment
directory
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1768041&r1=1768040&r2=1768041&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
Fri Nov 4 14:33:37 2016
@@ -18,6 +18,7 @@
*/
package org.apache.uima.ducc.ws;
+import java.util.HashSet;
import java.util.List;
import org.apache.uima.ducc.common.IDuccEnv;
@@ -52,6 +53,8 @@ public class DuccBoot extends Thread {
duccBoot.initialize(commonConfiguration);
duccBoot.start();
}
+
+ private HashSet<String> experimentsFound;
public void run() {
String location = "run";
@@ -128,11 +131,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of
Reservations fetched from history"), duccWorkReservations.size());
int restored = 0;
+ int nExperiments = 0;
for ( IDuccWorkReservation duccWorkReservation : duccWorkReservations
) {
try {
logger.debug(location, duccWorkReservation.getDuccId(),
messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkReservation);
- duccPlugins.restore(duccWorkReservation);
+ String directory =
duccWorkReservation.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkReservation);
+ nExperiments++;
+ }
restored++;
}
catch(Throwable t) {
@@ -140,13 +148,16 @@ public class DuccBoot extends Thread {
}
}
logger.info(location,null, messages.fetch("Reservations restored:
"+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found:
"+nExperiments));
+ }
}
private void restoreJobs(IHistoryPersistenceManager hpm, DuccData
duccData)
{
// Replaced for database. Both file and database now do all the
looping and sorting internally.
String location = "restoreJobs";
- List<IDuccWorkJob> duccWorkJobs = null;;
+ List<IDuccWorkJob> duccWorkJobs = null;
try {
duccWorkJobs = hpm.restoreJobs(maxJobs);
} catch (Exception e) {
@@ -157,11 +168,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of Jobs
fetched from history"), duccWorkJobs.size());
int restored = 0;
+ int nExperiments = 0;
for ( IDuccWorkJob duccWorkJob : duccWorkJobs ) {
try {
logger.debug(location, duccWorkJob.getDuccId(),
messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkJob);
- duccPlugins.restore(duccWorkJob);
+ String directory =
duccWorkJob.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkJob);
+ nExperiments++;
+ }
restored++;
}
catch(Throwable t) {
@@ -169,6 +185,9 @@ public class DuccBoot extends Thread {
}
}
logger.info(location,null, messages.fetch("Jobs restored: "+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found:
"+nExperiments));
+ }
}
// private void restoreJobsX(IHistoryPersistenceManager hpm, DuccData
duccData) {
@@ -214,11 +233,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of services
fetched from history"), duccWorkServices.size());
int restored = 0;
+ int nExperiments = 0;
for ( IDuccWorkService duccWorkService : duccWorkServices ) {
try {
logger.debug(location, duccWorkService.getDuccId(),
messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkService);
- duccPlugins.restore(duccWorkService);
+ String directory =
duccWorkService.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkService);
+ nExperiments++;
+ }
restored++;
}
catch(Throwable t) {
@@ -226,6 +250,9 @@ public class DuccBoot extends Thread {
}
}
logger.info(location,null, messages.fetch("Services restored:
"+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found:
"+nExperiments));
+ }
}
// private void restoreServices(IHistoryPersistenceManager hpm,
DuccData duccData) {
@@ -272,9 +299,11 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("History
directory")+IDuccEnv.DUCC_HISTORY_DIR);
IHistoryPersistenceManager hpm =
HistoryFactory.getInstance(this.getClass().getName());
DuccData duccData = DuccData.getInstance();
+ experimentsFound = new HashSet<String>(); // Lets the restore
methods avoid inspecting already-found experiments
restoreReservations(hpm, duccData);
restoreJobs(hpm, duccData);
restoreServices(hpm, duccData);
+ experimentsFound = null;
duccData.report();
}
}
