Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java Tue Sep 15 14:31:24 2015 @@ -26,4 +26,5 @@ public interface IDuccPerWorkItemStatist public double getMin(); public double getMean(); public double getStandardDeviation(); + public void setStandardDeviation(double s); }
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java Tue Sep 15 14:31:24 2015 @@ -18,15 +18,62 @@ */ package org.apache.uima.ducc.transport.event.common.history; +import org.apache.uima.ducc.common.main.DuccService; +import org.apache.uima.ducc.common.persistence.services.IStateServices; +import org.apache.uima.ducc.common.utils.DuccLogger; + public class HistoryFactory { - private static IHistoryPersistenceManager instance = new HistoryPersistenceManager(); + private static IHistoryPersistenceManager instance = null; //new HistoryPersistenceManager(); - public static IHistoryPersistenceManager getInstance() { - return instance; + public static IHistoryPersistenceManager getInstance(String callerClass) + { + if ( instance != null ) return instance; + + String methodName = "getInstance"; + + // log4j logging annoyance. We require the caller to give us its base package so + // we can configure a logger that writes to the right appender + int ndx = callerClass.lastIndexOf("."); + String stem = callerClass.substring(0, ndx); + + String clname = System.getProperty("ducc.job.history.impl"); + + if ( clname == null ) { + DuccLogger logger = DuccService.getDuccLogger(); + logger.error(methodName, null, "Job history class is not configured."); + instance = new NullHistoryManager(); + return instance; + } + + ndx = clname.lastIndexOf("."); + String clfile = clname.substring(ndx+1); + // + // We try to construct the persistence object. If it fails, we return a + // "null" object conforming to the interface but doing nothing to hopefully + // reduce NPEs. + // + DuccLogger logger = DuccLogger.getLogger(stem + "." + clfile, "DB"); // get the component logger + + // + // We try to construct the persistence object. If it fails, we return a + // "null" object conforming to the interface but doing nothing to hopefully + // reduce NPEs. + // + + try { + @SuppressWarnings("unchecked") + Class<IStateServices> iss = (Class<IStateServices>) Class.forName(clname); + instance = (IHistoryPersistenceManager) iss.newInstance(); + instance.setLogger(logger); + } catch ( Throwable t ) { + logger.error(methodName, null, "Cannot instantiate service persistence class", clname, ":", t); + instance = new NullHistoryManager(); + } + return instance; } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java Tue Sep 15 14:31:24 2015 @@ -25,7 +25,10 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.ListIterator; +import java.util.Map; import org.apache.uima.ducc.common.IDuccEnv; import org.apache.uima.ducc.common.main.DuccService; @@ -33,6 +36,7 @@ import org.apache.uima.ducc.common.utils import org.apache.uima.ducc.common.utils.IOHelper; import org.apache.uima.ducc.common.utils.Utils; import org.apache.uima.ducc.common.utils.id.DuccId; +import org.apache.uima.ducc.transport.event.common.DuccWorkMap; import org.apache.uima.ducc.transport.event.common.IDuccWorkJob; import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation; import org.apache.uima.ducc.transport.event.common.IDuccWorkService; @@ -40,353 +44,551 @@ import org.apache.uima.ducc.transport.ev public class HistoryPersistenceManager implements IHistoryPersistenceManager { - - // private static final DuccLogger logger = DuccLoggerComponents.getTrLogger(HistoryPersistenceManager.class.getName()); - private static final DuccLogger logger = DuccService.getDuccLogger(HistoryPersistenceManager.class.getName()); - - private String historyDirectory_jobs = IDuccEnv.DUCC_HISTORY_JOBS_DIR; - private String historyDirectory_reservations = IDuccEnv.DUCC_HISTORY_RESERVATIONS_DIR; - private String historyDirectory_services = IDuccEnv.DUCC_HISTORY_SERVICES_DIR; - - private String dwj = "dwj"; - private String dwr = "dwr"; - private String dws = "dws"; - - private enum Verbosity { - QUIET, - SPEAK, - } - - HistoryPersistenceManager() { - mkdirs(); - } - - private void mkdirs() { - IOHelper.mkdirs(historyDirectory_jobs); - IOHelper.mkdirs(historyDirectory_reservations); - IOHelper.mkdirs(historyDirectory_services); - } - - private String normalize(String id) { - String retVal = id; - return retVal; - } - - - public void jobSaveConditional(IDuccWorkJob duccWorkJob) throws IOException { - String id = normalize(""+duccWorkJob.getDuccId().getFriendly()); - String fileName = historyDirectory_jobs+File.separator+id+"."+dwj; - File file = new File(fileName); - if(!file.exists()) { - jobSave(duccWorkJob); - } - } - - - public void jobSave(IDuccWorkJob duccWorkJob) throws IOException { - String id = normalize(""+duccWorkJob.getDuccId().getFriendly()); - String fileName = historyDirectory_jobs+File.separator+id+"."+dwj; - FileOutputStream fos = null; - ObjectOutputStream out = null; - fos = new FileOutputStream(fileName); - out = new ObjectOutputStream(fos); - out.writeObject(duccWorkJob); - out.close(); - } - - public IDuccWorkJob jobRestore(String fileName) { - return jobRestore(fileName, Verbosity.SPEAK); - } - - private IDuccWorkJob jobRestore(String fileName, Verbosity level) { - String methodName = "jobRestore"; - IDuccWorkJob job = null; + + // private static final DuccLogger logger = DuccLoggerComponents.getTrLogger(HistoryPersistenceManager.class.getName()); + private DuccLogger logger = DuccService.getDuccLogger(HistoryPersistenceManager.class.getName()); + + private String historyDirectory_jobs = IDuccEnv.DUCC_HISTORY_JOBS_DIR; + private String historyDirectory_reservations = IDuccEnv.DUCC_HISTORY_RESERVATIONS_DIR; + private String historyDirectory_services = IDuccEnv.DUCC_HISTORY_SERVICES_DIR; + + private String dwj = "dwj"; + private String dwr = "dwr"; + private String dws = "dws"; + + private enum Verbosity { + QUIET, + SPEAK, + } + + HistoryPersistenceManager() { + mkdirs(); + } + + public void setLogger(DuccLogger logger) + { + this.logger = logger; + } + + private void mkdirs() { + IOHelper.mkdirs(historyDirectory_jobs); + IOHelper.mkdirs(historyDirectory_reservations); + IOHelper.mkdirs(historyDirectory_services); + } + + private String normalize(String id) { + String retVal = id; + return retVal; + } + + public void saveJob(IDuccWorkJob duccWorkJob) throws IOException { + String id = normalize(""+duccWorkJob.getDuccId().getFriendly()); + String fileName = historyDirectory_jobs+File.separator+id+"."+dwj; + FileOutputStream fos = null; + ObjectOutputStream out = null; + fos = new FileOutputStream(fileName); + out = new ObjectOutputStream(fos); + out.writeObject(duccWorkJob); + out.close(); + } + + public List<IDuccWorkJob> restoreJobs(long max) + throws Exception + { + String methodName = "restoreJobs"; + // Find the 'max' most current jobs. + // How: list all files; if ends with 'dwj' the first part is the duccid. + // put all the duccids into an array and sort it, (ascending so we don't need a comparator) + // Starting at the end, call restoreJob(Long) to get the file. + // + List<Long> allJobs = new ArrayList<Long>(); + File folder = new File(historyDirectory_jobs); + File[] listOfFiles = folder.listFiles(); + if(listOfFiles != null) { + for (int i = 0; i < listOfFiles.length; i++) { + if (listOfFiles[i].isFile()) { + String name = listOfFiles[i].getName(); + if(name.endsWith("."+dwj)) { + int ndx = name.indexOf("."); + try { + allJobs.add(Long.parseLong(name.substring(0, ndx))); + } catch ( NumberFormatException e ) { + logger.warn(methodName, null, "Job file", name, ": cannot extract ducc id. Not restored."); + } + + } + } + } + } + + max = Math.min(allJobs.size(), max); + Collections.sort(allJobs); + List<IDuccWorkJob> ret = new ArrayList<IDuccWorkJob>((int)max); + for ( int i = allJobs.size()-1; i >= 0; i-- ) { + ret.add(restoreJob(allJobs.get(i))); + } + + return ret; + } + + public IDuccWorkJob restoreJob(long duccid) + throws Exception + { + //String methodName = "jobRestore"; + IDuccWorkJob job = null; + String fileName = ""+duccid + "." + dwj; + FileInputStream fis = null; + ObjectInputStream in = null; + fis = new FileInputStream(historyDirectory_jobs+File.separator+fileName); + in = new ObjectInputStream(fis); + job = (IDuccWorkJob) in.readObject(); + in.close(); + return job; + } + + // public IDuccWorkJob jobRestore(String fileName) { + // return jobRestore(fileName, Verbosity.SPEAK); + // } + + // private IDuccWorkJob jobRestore(String fileName, Verbosity level) { + // String methodName = "jobRestore"; + // IDuccWorkJob job = null; + // try { + // logger.trace(methodName, null, "restore:"+fileName); + // FileInputStream fis = null; + // ObjectInputStream in = null; + // fis = new FileInputStream(historyDirectory_jobs+File.separator+fileName); + // in = new ObjectInputStream(fis); + // job = (IDuccWorkJob) in.readObject(); + // in.close(); + // } + // catch(Exception e) { + // switch(level) { + // case QUIET: + // break; + // case SPEAK: + // logger.warn(methodName, null, "unable to restore:"+fileName, e); + // break; + // } + // } + // return job; + // } + + + // public IDuccWorkJob jobRestore(DuccId duccId) { + // String fileName = duccId.getFriendly()+"."+dwj; + // return jobRestore(fileName, Verbosity.QUIET); + // } + + + // public ArrayList<Long> jobList() { return null; } +// public ArrayList<String> jobList() { +// ArrayList<String> retVal = new ArrayList<String>(); +// File folder = new File(historyDirectory_jobs); +// File[] listOfFiles = folder.listFiles(); +// if(listOfFiles != null) { +// for (int i = 0; i < listOfFiles.length; i++) { +// if (listOfFiles[i].isFile()) { +// String name = listOfFiles[i].getName(); +// if(name.endsWith("."+dwj)) { +// retVal.add(name); +// } +// } +// } +// } +// return retVal; +// } + + + // public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException { + // ArrayList<IDuccWorkJob> retVal = new ArrayList<IDuccWorkJob>(); + // ArrayList<String> jobFileNames = jobList(); + // ListIterator<String> listIterator = jobFileNames.listIterator(); + // while(listIterator.hasNext()) { + // String fileName = listIterator.next(); + // IDuccWorkJob job = jobRestore(fileName); + // if(job != null) { + // retVal.add(job); + // } + // } + // return retVal; + // } + + // public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException + // { + // return null; + // } + + +// public void reservationSaveConditional(IDuccWorkReservation duccWorkReservation) throws IOException { +// String id = normalize(""+duccWorkReservation.getDuccId().getFriendly()); +// String fileName = historyDirectory_jobs+File.separator+id+"."+dwr; +// File file = new File(fileName); +// if(!file.exists()) { +// reservationSave(duccWorkReservation); +// } +// } + + + public void saveReservation(IDuccWorkReservation duccWorkReservation) + throws Exception + { + String id = normalize(""+duccWorkReservation.getDuccId().getFriendly()); + String fileName = historyDirectory_reservations+File.separator+id+"."+dwr; + FileOutputStream fos = null; + ObjectOutputStream out = null; + fos = new FileOutputStream(fileName); + out = new ObjectOutputStream(fos); + out.writeObject(duccWorkReservation); + out.close(); + } + + + public IDuccWorkReservation restoreReservation(long duccid) + throws Exception + { + //String methodName = "reservationRestore"; + IDuccWorkReservation reservation = null; + FileInputStream fis = null; + ObjectInputStream in = null; + String fileName = ""+duccid + "." + dwr; + + fis = new FileInputStream(historyDirectory_reservations+File.separator+fileName); + in = new ObjectInputStream(fis); + reservation = (IDuccWorkReservation) in.readObject(); + in.close(); + return reservation; + } + + public List<IDuccWorkReservation> restoreReservations(long max) + throws Exception + { + String methodName = "restoreReservations"; + // Find the 'max' most current jobs. + // How: list all files; if ends with 'dwj' the first part is the duccid. + // put all the duccids into an array and sort it, (ascending so we don't need a comparator) + // Starting at the end, call restoreJob(Long) to get the file. + // + List<Long> allReservations = new ArrayList<Long>(); + File folder = new File(historyDirectory_reservations); + File[] listOfFiles = folder.listFiles(); + if(listOfFiles != null) { + for (int i = 0; i < listOfFiles.length; i++) { + if (listOfFiles[i].isFile()) { + String name = listOfFiles[i].getName(); + if(name.endsWith("."+dwr)) { + int ndx = name.indexOf("."); + try { + allReservations.add(Long.parseLong(name.substring(0, ndx))); + } catch ( NumberFormatException e ) { + logger.warn(methodName, null, "Reservation file", name, ": cannot extract ducc id. Not restored."); + } + + } + } + } + } + + max = Math.min(allReservations.size(), max); + Collections.sort(allReservations); + List<IDuccWorkReservation> ret = new ArrayList<IDuccWorkReservation>((int)max); + for ( int i = allReservations.size()-1; i >= 0; i-- ) { + ret.add(restoreReservation(allReservations.get(i))); + } + + return ret; + } + + + // private IDuccWorkReservation reservationRestore(String fileName, Verbosity level) { + // String methodName = "reservationRestore"; + // IDuccWorkReservation reservation = null; + // try { + // logger.trace(methodName, null, "restore:"+fileName); + // FileInputStream fis = null; + // ObjectInputStream in = null; + // fis = new FileInputStream(historyDirectory_reservations+File.separator+fileName); + // in = new ObjectInputStream(fis); + // reservation = (IDuccWorkReservation) in.readObject(); + // in.close(); + // } + // catch(Exception e) { + // switch(level) { + // case QUIET: + // break; + // case SPEAK: + // logger.warn(methodName, null, "unable to restore:"+fileName); + // break; + // } + // } + // return reservation; + // } + + + // public ArrayList<String> reservationList() { + // ArrayList<String> retVal = new ArrayList<String>(); + // File folder = new File(historyDirectory_reservations); + // File[] listOfFiles = folder.listFiles(); + // if(listOfFiles != null) { + // for (int i = 0; i < listOfFiles.length; i++) { + // if (listOfFiles[i].isFile()) { + // String name = listOfFiles[i].getName(); + // if(name.endsWith("."+dwr)) { + // retVal.add(name); + // } + // } + // } + // } + // return retVal; + // } + + + // public ArrayList<IDuccWorkReservation> reservationRestore() throws IOException, ClassNotFoundException { + // ArrayList<IDuccWorkReservation> retVal = new ArrayList<IDuccWorkReservation>(); + // ArrayList<String> reservationFileNames = reservationList(); + // ListIterator<String> listIterator = reservationFileNames.listIterator(); + // while(listIterator.hasNext()) { + // String fileName = listIterator.next(); + // IDuccWorkReservation reservation = reservationRestore(fileName); + // if(reservation != null) { + // retVal.add(reservation); + // } + // } + // return retVal; + // } + + + // public IDuccWorkReservation reservationRestore(DuccId duccId) { + // String fileName = duccId.getFriendly()+"."+dwr; + // return reservationRestore(fileName, Verbosity.QUIET); + // } + + + public void saveService(IDuccWorkService duccWorkService) + throws Exception + { + String id = normalize(""+duccWorkService.getDuccId().getFriendly()); + String fileName = historyDirectory_services+File.separator+id+"."+dws; + FileOutputStream fos = null; + ObjectOutputStream out = null; + fos = new FileOutputStream(fileName); + out = new ObjectOutputStream(fos); + out.writeObject(duccWorkService); + out.close(); + } + + + public IDuccWorkService restoreService(long duccid) + throws Exception + { + //String methodName = "restoreService"; + IDuccWorkService service = null; + FileInputStream fis = null; + ObjectInputStream in = null; + String fileName = ""+duccid + "." + dws; + + fis = new FileInputStream(historyDirectory_services+File.separator+fileName); + in = new ObjectInputStream(fis); + service = (IDuccWorkService) in.readObject(); + in.close(); + return service; + } + + public List<IDuccWorkService> restoreServices(long max) + throws Exception + { + String methodName = "restoreServices"; + // Find the 'max' most current jobs. + // How: list all files; if ends with 'dwj' the first part is the duccid. + // put all the duccids into an array and sort it, (ascending so we don't need a comparator) + // Starting at the end, call restoreJob(Long) to get the file. + // + List<Long> allServices = new ArrayList<Long>(); + File folder = new File(historyDirectory_services); + File[] listOfFiles = folder.listFiles(); + if(listOfFiles != null) { + for (int i = 0; i < listOfFiles.length; i++) { + if (listOfFiles[i].isFile()) { + String name = listOfFiles[i].getName(); + if(name.endsWith("."+dws)) { + int ndx = name.indexOf("."); + try { + allServices.add(Long.parseLong(name.substring(0, ndx))); + } catch ( NumberFormatException e ) { + logger.warn(methodName, null, "Reservation file", name, ": cannot extract ducc id. Not restored."); + } + + } + } + } + } + + max = Math.min(allServices.size(), max); + Collections.sort(allServices); + List<IDuccWorkService> ret = new ArrayList<IDuccWorkService>((int)max); + for ( int i = allServices.size()-1; i >= 0; i-- ) { + ret.add(restoreService(allServices.get(i))); + } + + return ret; + } + + // public void serviceSaveConditional(IDuccWorkService duccWorkService) + // throws IOException { + // String id = normalize(""+duccWorkService.getDuccId().getFriendly()); + // String fileName = historyDirectory_services+File.separator+id+"."+dws; + // File file = new File(fileName); + // if(!file.exists()) { + // serviceSave(duccWorkService); + // } + // } + + + + public void serviceSave(IDuccWorkService duccWorkService) + throws IOException { + String id = normalize(""+duccWorkService.getDuccId().getFriendly()); + String fileName = historyDirectory_services+File.separator+id+"."+dws; + FileOutputStream fos = null; + ObjectOutputStream out = null; + fos = new FileOutputStream(fileName); + out = new ObjectOutputStream(fos); + out.writeObject(duccWorkService); + out.close(); + } + + + public IDuccWorkService serviceRestore(String fileName) { + return serviceRestore(fileName, Verbosity.SPEAK); + } + + private IDuccWorkService serviceRestore(String fileName, Verbosity level) { + String methodName = "serviceRestore"; + IDuccWorkService service = null; + try { + logger.trace(methodName, null, "restore:"+fileName); + FileInputStream fis = null; + ObjectInputStream in = null; + fis = new FileInputStream(historyDirectory_services+File.separator+fileName); + in = new ObjectInputStream(fis); + service = (IDuccWorkService) in.readObject(); + in.close(); + } + catch(Exception e) { + switch(level) { + case QUIET: + break; + case SPEAK: + logger.warn(methodName, null, "unable to restore:"+fileName); + break; + } + } + return service; + } + + + public ArrayList<String> serviceList() { + ArrayList<String> retVal = new ArrayList<String>(); + File folder = new File(historyDirectory_services); + File[] listOfFiles = folder.listFiles(); + if(listOfFiles != null) { + for (int i = 0; i < listOfFiles.length; i++) { + if (listOfFiles[i].isFile()) { + String name = listOfFiles[i].getName(); + if(name.endsWith("."+dws)) { + retVal.add(name); + } + } + } + } + return retVal; + } + + + public ArrayList<IDuccWorkService> serviceRestore() throws IOException, + ClassNotFoundException { + ArrayList<IDuccWorkService> retVal = new ArrayList<IDuccWorkService>(); + ArrayList<String> serviceFileNames = serviceList(); + ListIterator<String> listIterator = serviceFileNames.listIterator(); + while(listIterator.hasNext()) { + String fileName = listIterator.next(); + IDuccWorkService service = serviceRestore(fileName); + if(service != null) { + retVal.add(service); + } + } + return retVal; + } + + + public IDuccWorkService serviceRestore(DuccId duccId) { + String fileName = duccId.getFriendly()+"."+dws; + return serviceRestore(fileName, Verbosity.QUIET); + } + + public boolean checkpoint(DuccWorkMap m, Map<DuccId, DuccId> processToJob) + throws Exception + { + return false; + } + + public boolean restore(DuccWorkMap m, Map<DuccId, DuccId> processToJob) + throws Exception + { + return false; + } + + private static int doJobs(HistoryPersistenceManager hpm) + throws Exception + { + List<IDuccWorkJob> duccWorkJobs = hpm.restoreJobs(-1L); + ListIterator<IDuccWorkJob> listIterator = duccWorkJobs.listIterator(); + int acc = 0; + while(listIterator.hasNext()) { + IDuccWorkJob duccWorkJob = listIterator.next(); + System.out.println(duccWorkJob.getId()); + acc++; + } + return acc; + } + + private static int doReservations(HistoryPersistenceManager hpm) + throws Exception + { + List<IDuccWorkReservation> duccWorkReservations = hpm.restoreReservations(-1L); + ListIterator<IDuccWorkReservation> listIterator = duccWorkReservations.listIterator(); + int acc = 0; + while(listIterator.hasNext()) { + IDuccWorkReservation duccWorkReservation = listIterator.next(); + System.out.println(duccWorkReservation.getId()); + acc++; + } + return acc; + } + + public static void main(String[] args) throws IOException, ClassNotFoundException { + String ducc_home = Utils.findDuccHome(); + if(ducc_home == null) { + System.out.println("DUCC_HOME not set in environment"); + return; + } + if(ducc_home.trim() == "") { + System.out.println("DUCC_HOME not set in environment"); + return; + } + int reservations = 0; try { - logger.trace(methodName, null, "restore:"+fileName); - FileInputStream fis = null; - ObjectInputStream in = null; - fis = new FileInputStream(historyDirectory_jobs+File.separator+fileName); - in = new ObjectInputStream(fis); - job = (IDuccWorkJob) in.readObject(); - in.close(); - } - catch(Exception e) { - switch(level) { - case QUIET: - break; - case SPEAK: - logger.warn(methodName, null, "unable to restore:"+fileName, e); - break; - } - } - return job; - } - - - public IDuccWorkJob jobRestore(DuccId duccId) { - String fileName = duccId.getFriendly()+"."+dwj; - return jobRestore(fileName, Verbosity.QUIET); - } - - - public ArrayList<String> jobList() { - ArrayList<String> retVal = new ArrayList<String>(); - File folder = new File(historyDirectory_jobs); - File[] listOfFiles = folder.listFiles(); - if(listOfFiles != null) { - for (int i = 0; i < listOfFiles.length; i++) { - if (listOfFiles[i].isFile()) { - String name = listOfFiles[i].getName(); - if(name.endsWith("."+dwj)) { - retVal.add(name); - } - } - } - } - return retVal; - } - - - public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException { - ArrayList<IDuccWorkJob> retVal = new ArrayList<IDuccWorkJob>(); - ArrayList<String> jobFileNames = jobList(); - ListIterator<String> listIterator = jobFileNames.listIterator(); - while(listIterator.hasNext()) { - String fileName = listIterator.next(); - IDuccWorkJob job = jobRestore(fileName); - if(job != null) { - retVal.add(job); - } - } - return retVal; - } - - - public void reservationSaveConditional(IDuccWorkReservation duccWorkReservation) throws IOException { - String id = normalize(""+duccWorkReservation.getDuccId().getFriendly()); - String fileName = historyDirectory_jobs+File.separator+id+"."+dwr; - File file = new File(fileName); - if(!file.exists()) { - reservationSave(duccWorkReservation); - } - } - - - public void reservationSave(IDuccWorkReservation duccWorkReservation) throws IOException { - String id = normalize(""+duccWorkReservation.getDuccId().getFriendly()); - String fileName = historyDirectory_reservations+File.separator+id+"."+dwr; - FileOutputStream fos = null; - ObjectOutputStream out = null; - fos = new FileOutputStream(fileName); - out = new ObjectOutputStream(fos); - out.writeObject(duccWorkReservation); - out.close(); - } - - - public IDuccWorkReservation reservationRestore(String fileName) { - return reservationRestore(fileName, Verbosity.SPEAK); - } - - private IDuccWorkReservation reservationRestore(String fileName, Verbosity level) { - String methodName = "reservationRestore"; - IDuccWorkReservation reservation = null; - try { - logger.trace(methodName, null, "restore:"+fileName); - FileInputStream fis = null; - ObjectInputStream in = null; - fis = new FileInputStream(historyDirectory_reservations+File.separator+fileName); - in = new ObjectInputStream(fis); - reservation = (IDuccWorkReservation) in.readObject(); - in.close(); - } - catch(Exception e) { - switch(level) { - case QUIET: - break; - case SPEAK: - logger.warn(methodName, null, "unable to restore:"+fileName); - break; - } - } - return reservation; - } - - - public ArrayList<String> reservationList() { - ArrayList<String> retVal = new ArrayList<String>(); - File folder = new File(historyDirectory_reservations); - File[] listOfFiles = folder.listFiles(); - if(listOfFiles != null) { - for (int i = 0; i < listOfFiles.length; i++) { - if (listOfFiles[i].isFile()) { - String name = listOfFiles[i].getName(); - if(name.endsWith("."+dwr)) { - retVal.add(name); - } - } - } - } - return retVal; - } - - - public ArrayList<IDuccWorkReservation> reservationRestore() throws IOException, ClassNotFoundException { - ArrayList<IDuccWorkReservation> retVal = new ArrayList<IDuccWorkReservation>(); - ArrayList<String> reservationFileNames = reservationList(); - ListIterator<String> listIterator = reservationFileNames.listIterator(); - while(listIterator.hasNext()) { - String fileName = listIterator.next(); - IDuccWorkReservation reservation = reservationRestore(fileName); - if(reservation != null) { - retVal.add(reservation); - } - } - return retVal; - } - - - public IDuccWorkReservation reservationRestore(DuccId duccId) { - String fileName = duccId.getFriendly()+"."+dwr; - return reservationRestore(fileName, Verbosity.QUIET); - } - - - public void serviceSaveConditional(IDuccWorkService duccWorkService) - throws IOException { - String id = normalize(""+duccWorkService.getDuccId().getFriendly()); - String fileName = historyDirectory_services+File.separator+id+"."+dws; - File file = new File(fileName); - if(!file.exists()) { - serviceSave(duccWorkService); - } - } - - - public void serviceSave(IDuccWorkService duccWorkService) - throws IOException { - String id = normalize(""+duccWorkService.getDuccId().getFriendly()); - String fileName = historyDirectory_services+File.separator+id+"."+dws; - FileOutputStream fos = null; - ObjectOutputStream out = null; - fos = new FileOutputStream(fileName); - out = new ObjectOutputStream(fos); - out.writeObject(duccWorkService); - out.close(); - } - - - public IDuccWorkService serviceRestore(String fileName) { - return serviceRestore(fileName, Verbosity.SPEAK); - } - - private IDuccWorkService serviceRestore(String fileName, Verbosity level) { - String methodName = "serviceRestore"; - IDuccWorkService service = null; - try { - logger.trace(methodName, null, "restore:"+fileName); - FileInputStream fis = null; - ObjectInputStream in = null; - fis = new FileInputStream(historyDirectory_services+File.separator+fileName); - in = new ObjectInputStream(fis); - service = (IDuccWorkService) in.readObject(); - in.close(); - } - catch(Exception e) { - switch(level) { - case QUIET: - break; - case SPEAK: - logger.warn(methodName, null, "unable to restore:"+fileName); - break; - } - } - return service; - } - - - public ArrayList<String> serviceList() { - ArrayList<String> retVal = new ArrayList<String>(); - File folder = new File(historyDirectory_services); - File[] listOfFiles = folder.listFiles(); - if(listOfFiles != null) { - for (int i = 0; i < listOfFiles.length; i++) { - if (listOfFiles[i].isFile()) { - String name = listOfFiles[i].getName(); - if(name.endsWith("."+dws)) { - retVal.add(name); - } - } - } - } - return retVal; - } - - - public ArrayList<IDuccWorkService> serviceRestore() throws IOException, - ClassNotFoundException { - ArrayList<IDuccWorkService> retVal = new ArrayList<IDuccWorkService>(); - ArrayList<String> serviceFileNames = serviceList(); - ListIterator<String> listIterator = serviceFileNames.listIterator(); - while(listIterator.hasNext()) { - String fileName = listIterator.next(); - IDuccWorkService service = serviceRestore(fileName); - if(service != null) { - retVal.add(service); - } - } - return retVal; - } - - - public IDuccWorkService serviceRestore(DuccId duccId) { - String fileName = duccId.getFriendly()+"."+dws; - return serviceRestore(fileName, Verbosity.QUIET); - } - - ///// <tool> - - private static int doJobs(HistoryPersistenceManager hpm) throws IOException, ClassNotFoundException { - ArrayList<IDuccWorkJob> duccWorkJobs = hpm.jobRestore(); - ListIterator<IDuccWorkJob> listIterator = duccWorkJobs.listIterator(); - int acc = 0; - while(listIterator.hasNext()) { - IDuccWorkJob duccWorkJob = listIterator.next(); - System.out.println(duccWorkJob.getId()); - acc++; - } - return acc; - } - - private static int doReservations(HistoryPersistenceManager hpm) throws IOException, ClassNotFoundException { - ArrayList<IDuccWorkReservation> duccWorkReservations = hpm.reservationRestore(); - ListIterator<IDuccWorkReservation> listIterator = duccWorkReservations.listIterator(); - int acc = 0; - while(listIterator.hasNext()) { - IDuccWorkReservation duccWorkReservation = listIterator.next(); - System.out.println(duccWorkReservation.getId()); - acc++; - } - return acc; - } - - public static void main(String[] args) throws IOException, ClassNotFoundException { - String ducc_home = Utils.findDuccHome(); - if(ducc_home == null) { - System.out.println("DUCC_HOME not set in environment"); - return; - } - if(ducc_home.trim() == "") { - System.out.println("DUCC_HOME not set in environment"); - return; + HistoryPersistenceManager hpm = new HistoryPersistenceManager(); + int jobs = doJobs(hpm); + System.out.println("jobs: "+jobs); + reservations = doReservations(hpm); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); } - HistoryPersistenceManager hpm = new HistoryPersistenceManager(); - int jobs = doJobs(hpm); - System.out.println("jobs: "+jobs); - int reservations = doReservations(hpm); - System.out.println("reservations: "+reservations); - } + System.out.println("reservations: "+reservations); + } - ///// </tool> + ///// </tool> } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java Tue Sep 15 14:31:24 2015 @@ -18,35 +18,39 @@ */ package org.apache.uima.ducc.transport.event.common.history; -import java.io.IOException; -import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.uima.ducc.common.utils.DuccLogger; import org.apache.uima.ducc.common.utils.id.DuccId; +import org.apache.uima.ducc.transport.event.common.DuccWorkMap; import org.apache.uima.ducc.transport.event.common.IDuccWorkJob; import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation; import org.apache.uima.ducc.transport.event.common.IDuccWorkService; -public interface IHistoryPersistenceManager { - - public void jobSaveConditional(IDuccWorkJob duccWorkJob) throws IOException; - public void jobSave(IDuccWorkJob duccWorkJob) throws IOException; - public IDuccWorkJob jobRestore(String fileName); - public IDuccWorkJob jobRestore(DuccId duccId); - public ArrayList<String> jobList(); - public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException; - - public void reservationSaveConditional(IDuccWorkReservation duccWorkReservation) throws IOException; - public void reservationSave(IDuccWorkReservation duccWorkReservation) throws IOException; - public IDuccWorkReservation reservationRestore(String fileName); - public IDuccWorkReservation reservationRestore(DuccId duccId); - public ArrayList<String> reservationList(); - public ArrayList<IDuccWorkReservation> reservationRestore() throws IOException, ClassNotFoundException; +public interface IHistoryPersistenceManager +{ + public void saveJob(IDuccWorkJob duccWorkJob) throws Exception; + public IDuccWorkJob restoreJob(long friendly_id) throws Exception; + public List<IDuccWorkJob> restoreJobs(long max) throws Exception; - public void serviceSaveConditional(IDuccWorkService duccWorkService) throws IOException; - public void serviceSave(IDuccWorkService duccWorkService) throws IOException; - public IDuccWorkService serviceRestore(String fileName); - public IDuccWorkService serviceRestore(DuccId duccId); - public ArrayList<String> serviceList(); - public ArrayList<IDuccWorkService> serviceRestore() throws IOException, ClassNotFoundException; + public void saveReservation(IDuccWorkReservation reservation) throws Exception; + public IDuccWorkReservation restoreReservation(long friendly_id) throws Exception; + public List<IDuccWorkReservation> restoreReservations(long max) throws Exception; + + public void saveService(IDuccWorkService duccWorkService) throws Exception; + public IDuccWorkService restoreService(long friendly_id) throws Exception; + public List<IDuccWorkService> restoreServices(long max) throws Exception; + + public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception; + public boolean restore(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception; + + public void setLogger(DuccLogger logger); + //public void serviceSaveConditional(IDuccWorkService duccWorkService) throws Exception; + // public void serviceSave(IDuccWorkService duccWorkService) throws Exception; + //public IDuccWorkService serviceRestore(String fileName); + //public IDuccWorkService serviceRestore(DuccId duccId); + //public ArrayList<String> serviceList(); + //public ArrayList<IDuccWorkService> serviceRestore() throws IOException, ClassNotFoundException; } Added: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java?rev=1703203&view=auto ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java (added) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java Tue Sep 15 14:31:24 2015 @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +*/ +package org.apache.uima.ducc.transport.event.common.history; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.uima.ducc.common.utils.DuccLogger; +import org.apache.uima.ducc.common.utils.id.DuccId; +import org.apache.uima.ducc.transport.event.common.DuccWorkMap; +import org.apache.uima.ducc.transport.event.common.IDuccWorkJob; +import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation; +import org.apache.uima.ducc.transport.event.common.IDuccWorkService; + + +public class NullHistoryManager + implements IHistoryPersistenceManager +{ + NullHistoryManager() + { + } + + public void setLogger(DuccLogger logger) {} + + public void saveJob(IDuccWorkJob duccWorkJob) + throws Exception + { + } + + public IDuccWorkJob restoreJob(long duccid) + throws Exception + { + return null; + } + + public List<IDuccWorkJob> restoreJobs(long max) + throws Exception + { + return new ArrayList<IDuccWorkJob>(); + } + + + public void saveReservation(IDuccWorkReservation reservation) + throws Exception + { + } + + public IDuccWorkReservation restoreReservation(long friendly_id) + throws Exception + { + return null; + } + + public List<IDuccWorkReservation> restoreReservations(long max) + throws Exception + { + return new ArrayList<IDuccWorkReservation>(); + } + + + public void saveService(IDuccWorkService service) + throws Exception + { + } + + + public void serviceSave(IDuccWorkService service) + throws Exception + { + } + + + public IDuccWorkService restoreService(long duccid) + throws Exception + { + return null; + } + + public List<IDuccWorkService> restoreServices(long max) + throws Exception + { + return new ArrayList<IDuccWorkService>(); + } + + + public IDuccWorkService serviceRestore(String fileName) + { + return null; + } + + public ArrayList<String> serviceList() + { + return new ArrayList<String>(); + } + + + public ArrayList<IDuccWorkService> serviceRestore() + throws IOException, + ClassNotFoundException + { + return new ArrayList<IDuccWorkService>(); + } + + + public IDuccWorkService serviceRestore(DuccId duccId) + { + return null; + } + + public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception { return false; } + public boolean restore(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception { return false; } + +} Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Tue Sep 15 14:31:24 2015 @@ -19,7 +19,7 @@ package org.apache.uima.ducc.ws; import java.util.ArrayList; -import java.util.Iterator; +import java.util.List; import java.util.ListIterator; import java.util.TreeMap; @@ -105,89 +105,176 @@ public class DuccBoot extends Thread { return map; } - private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) { - String location = "restoreReservations"; - ArrayList<String> duccWorkReservations = hpm.reservationList(); - logger.info(location, jobid, messages.fetchLabel("Number of Reservations to restore")+duccWorkReservations.size()); - TreeMap<Integer,String> map = sort(duccWorkReservations); - Iterator<Integer> iterator = map.descendingKeySet().iterator(); - int i = 0; - int restored = 0; - while(iterator.hasNext() && (++i < maxReservations)) { - try { - Integer key = iterator.next(); - logger.debug(location, jobid, messages.fetchLabel("restore")+key); - String fileName = map.get(key); - IDuccWorkReservation duccWorkReservation; - duccWorkReservation = hpm.reservationRestore(fileName); - if(duccWorkReservation != null) { - duccData.putIfNotPresent(duccWorkReservation); - duccPlugins.restore(duccWorkReservation); - restored++; - } - } - catch(Throwable t) { - logger.warn(location, jobid, t); - } + // private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) { + // String location = "restoreReservations"; + // ArrayList<String> duccWorkReservations = hpm.reservationList(); + // logger.info(location, jobid, messages.fetchLabel("Number of Reservations to restore")+duccWorkReservations.size()); + // TreeMap<Integer,String> map = sort(duccWorkReservations); + // Iterator<Integer> iterator = map.descendingKeySet().iterator(); + // int i = 0; + // int restored = 0; + // while(iterator.hasNext() && (++i < maxReservations)) { + // try { + // Integer key = iterator.next(); + // logger.debug(location, jobid, messages.fetchLabel("restore")+key); + // String fileName = map.get(key); + // IDuccWorkReservation duccWorkReservation; + // duccWorkReservation = hpm.reservationRestore(fileName); + // if(duccWorkReservation != null) { + // duccData.putIfNotPresent(duccWorkReservation); + // duccPlugins.restore(duccWorkReservation); + // restored++; + // } + // } + // catch(Throwable t) { + // logger.warn(location, jobid, t); + // } + // } + // logger.info(location, jobid, messages.fetch("Reservations restored: "+restored)); + // } + + private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) + { + // Replaced for database. Both file and database now do all the looping and sorting internally. + String location = "restoreReservations"; + List<IDuccWorkReservation> duccWorkReservations = null;; + try { + duccWorkReservations = hpm.restoreReservations(maxReservations); + } catch (Exception e) { + logger.warn(location, null, e); + return; // Nothing to do if this fails } - logger.info(location, jobid, messages.fetch("Reservations restored: "+restored)); - } - - private void restoreJobs(IHistoryPersistenceManager hpm, DuccData duccData) { - String location = "restoreJobs"; - ArrayList<String> duccWorkJobs = hpm.jobList(); - logger.info(location, jobid, messages.fetchLabel("Number of Jobs to restore")+duccWorkJobs.size()); - TreeMap<Integer,String> map = sort(duccWorkJobs); - Iterator<Integer> iterator = map.descendingKeySet().iterator(); - int i = 0; - int restored = 0; - while(iterator.hasNext() && (++i < maxJobs)) { - try { - Integer key = iterator.next(); - logger.debug(location, jobid, messages.fetchLabel("restore")+key); - String fileName = map.get(key); - IDuccWorkJob duccWorkJob; - duccWorkJob = hpm.jobRestore(fileName); - if(duccWorkJob != null) { - duccData.putIfNotPresent(duccWorkJob); - duccPlugins.restore(duccWorkJob); - restored++; - } - } - catch(Throwable t) { - logger.warn(location, jobid, t); - } + + logger.info(location, jobid, messages.fetchLabel("Number of Reservations fetched from DB"), duccWorkReservations.size()); + + int restored = 0; + for ( IDuccWorkReservation duccWorkReservation : duccWorkReservations ) { + try { + logger.debug(location, duccWorkReservation.getDuccId(), messages.fetchLabel("restore")); + duccData.putIfNotPresent(duccWorkReservation); + duccPlugins.restore(duccWorkReservation); + restored++; + } + catch(Throwable t) { + logger.warn(location, duccWorkReservation.getDuccId(), t); + } + } + logger.info(location,null, messages.fetch("Reservations restored: "+restored)); + } + + private void restoreJobs(IHistoryPersistenceManager hpm, DuccData duccData) + { + // Replaced for database. Both file and database now do all the looping and sorting internally. + String location = "restoreJobs"; + List<IDuccWorkJob> duccWorkJobs = null;; + try { + duccWorkJobs = hpm.restoreJobs(maxJobs); + } catch (Exception e) { + logger.warn(location, null, e); + return; // Nothing to do if this fails } - logger.info(location, jobid, messages.fetch("Jobs restored: "+restored)); - } - - private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) { - String location = "restoreServices"; - ArrayList<String> duccWorkServices = hpm.serviceList(); - logger.info(location, jobid, messages.fetchLabel("Number of Services to restore")+duccWorkServices.size()); - TreeMap<Integer,String> map = sort(duccWorkServices); - Iterator<Integer> iterator = map.descendingKeySet().iterator(); - int i = 0; - int restored = 0; - while(iterator.hasNext() && (++i < maxServices)) { - try { - Integer key = iterator.next(); - logger.debug(location, jobid, messages.fetchLabel("restore")+key); - String fileName = map.get(key); - IDuccWorkService duccWorkService; - duccWorkService = hpm.serviceRestore(fileName); - if(duccWorkService != null) { - duccData.putIfNotPresent(duccWorkService); - duccPlugins.restore(duccWorkService); - restored++; - } - } - catch(Throwable t) { - logger.warn(location, jobid, t); - } + + logger.info(location, jobid, messages.fetchLabel("Number of Jobs fetched from DB"), duccWorkJobs.size()); + + int restored = 0; + for ( IDuccWorkJob duccWorkJob : duccWorkJobs ) { + try { + logger.debug(location, duccWorkJob.getDuccId(), messages.fetchLabel("restore")); + duccData.putIfNotPresent(duccWorkJob); + duccPlugins.restore(duccWorkJob); + restored++; + } + catch(Throwable t) { + logger.warn(location, duccWorkJob.getDuccId(), t); + } + } + logger.info(location,null, messages.fetch("Jobs restored: "+restored)); + } + +// private void restoreJobsX(IHistoryPersistenceManager hpm, DuccData duccData) { +// String location = "restoreJobs"; +// ArrayList<String> duccWorkJobs = hpm.jobList(); +// logger.info(location, jobid, messages.fetchLabel("Number of Jobs to restore")+duccWorkJobs.size()); +// TreeMap<Integer,String> map = sort(duccWorkJobs); +// Iterator<Integer> iterator = map.descendingKeySet().iterator(); +// int i = 0; +// int restored = 0; +// while(iterator.hasNext() && (++i < maxJobs)) { +// try { +// Integer key = iterator.next(); +// logger.debug(location, jobid, messages.fetchLabel("restore")+key); +// String fileName = map.get(key); +// IDuccWorkJob duccWorkJob; +// duccWorkJob = hpm.jobRestore(fileName); +// if(duccWorkJob != null) { +// duccData.putIfNotPresent(duccWorkJob); +// duccPlugins.restore(duccWorkJob); +// restored++; +// } +// } +// catch(Throwable t) { +// logger.warn(location, jobid, t); +// } +// } +// logger.info(location, jobid, messages.fetch("Jobs restored: "+restored)); +// } + + private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) + { + // Replaced for database. Both file and database now do all the looping and sorting internally. + String location = "restoreServices"; + List<IDuccWorkService> duccWorkServices = null;; + try { + duccWorkServices = hpm.restoreServices(maxServices); + } catch (Exception e) { + logger.warn(location, null, e); + return; // Nothing to do if this fails } - logger.info(location, jobid, messages.fetch("Services restored: "+restored)); - } + + logger.info(location, jobid, messages.fetchLabel("Number of services fetched from DB"), duccWorkServices.size()); + + int restored = 0; + for ( IDuccWorkService duccWorkService : duccWorkServices ) { + try { + logger.debug(location, duccWorkService.getDuccId(), messages.fetchLabel("restore")); + duccData.putIfNotPresent(duccWorkService); + duccPlugins.restore(duccWorkService); + restored++; + } + catch(Throwable t) { + logger.warn(location, duccWorkService.getDuccId(), t); + } + } + logger.info(location,null, messages.fetch("Services restored: "+restored)); + } + + // private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) { + // String location = "restoreServices"; + // ArrayList<String> duccWorkServices = hpm.serviceList(); + // logger.info(location, jobid, messages.fetchLabel("Number of Services to restore")+duccWorkServices.size()); + // TreeMap<Integer,String> map = sort(duccWorkServices); + // Iterator<Integer> iterator = map.descendingKeySet().iterator(); + // int i = 0; + // int restored = 0; + // while(iterator.hasNext() && (++i < maxServices)) { + // try { + // Integer key = iterator.next(); + // logger.debug(location, jobid, messages.fetchLabel("restore")+key); + // String fileName = map.get(key); + // IDuccWorkService duccWorkService; + // duccWorkService = hpm.serviceRestore(fileName); + // if(duccWorkService != null) { + // duccData.putIfNotPresent(duccWorkService); + // duccPlugins.restore(duccWorkService); + // restored++; + // } + // } + // catch(Throwable t) { + // logger.warn(location, jobid, t); + // } + // } + // logger.info(location, jobid, messages.fetch("Services restored: "+restored)); + // } private void initialize(CommonConfiguration commonConfiguration) { String location = "initialize"; @@ -203,7 +290,7 @@ public class DuccBoot extends Thread { private void restore() { String location = "restore"; logger.info(location, jobid, messages.fetchLabel("History directory")+IDuccEnv.DUCC_HISTORY_DIR); - IHistoryPersistenceManager hpm = HistoryFactory.getInstance(); + IHistoryPersistenceManager hpm = HistoryFactory.getInstance(this.getClass().getName()); DuccData duccData = DuccData.getInstance(); restoreReservations(hpm, duccData); restoreJobs(hpm, duccData); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java Tue Sep 15 14:31:24 2015 @@ -70,7 +70,7 @@ public class DuccData { private volatile String published = null; - private IHistoryPersistenceManager hpm = HistoryFactory.getInstance(); + private IHistoryPersistenceManager hpm = HistoryFactory.getInstance(this.getClass().getName()); public boolean isPublished() { return published != null; @@ -95,26 +95,32 @@ public class DuccData { } @SuppressWarnings("unchecked") - private void mergeHistory(IDuccWorkMap map) { + private void mergeHistory(IDuccWorkMap map) + { + String methodName = "mergeHistory"; Iterator<DuccId> iterator = duccWorkLive.keySet().iterator(); while(iterator.hasNext()) { DuccId duccId = iterator.next(); IDuccWork duccWork = duccWorkLive.findDuccWork(duccId); IDuccWork history = null; - switch(duccWork.getDuccType()) { - case Job: - history = hpm.jobRestore(duccId); - break; - case Reservation: - history = hpm.reservationRestore(duccId); - break; - case Service: - history = hpm.serviceRestore(duccId); - break; - } - if(history != null) { - map.put(duccId, history); - } + try { + switch(duccWork.getDuccType()) { + case Job: + history = hpm.restoreJob(duccId.getFriendly()); + break; + case Reservation: + history = hpm.restoreReservation(duccId.getFriendly()); + break; + case Service: + history = hpm.restoreService(duccId.getFriendly()); + break; + } + if(history != null) { + map.put(duccId, history); + } + } catch ( Exception e ) { + logger.warn(methodName, duccId, "Cannot recover", duccWork.getDuccType(), "from database"); + } } } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java Tue Sep 15 14:31:24 2015 @@ -26,28 +26,28 @@ public class IServicesRegistry { public static final String svc = IStateServices.svc; // meta - public static final String autostart = IStateServices.autostart; - public static final String reference = IStateServices.reference; - public static final String enabled = IStateServices.enabled; - public static final String disable_reason = IStateServices.disable_reason; - public static final String endpoint = IStateServices.endpoint; - public static final String implementors = IStateServices.implementors; - public static final String instances = IStateServices.instances; - public static final String numeric_id = IStateServices.numeric_id; - public static final String ping_active = IStateServices.ping_active; - public static final String ping_only = IStateServices.ping_only; - public static final String service_alive = IStateServices.service_alive; - public static final String service_class = IStateServices.service_class; - public static final String service_dependency = IStateServices.service_dependency; - public static final String service_healthy = IStateServices.service_healthy; - public static final String service_state = IStateServices.service_state; - public static final String last_use = IStateServices.last_use; - public static final String service_statistics = IStateServices.service_statistics; - public static final String service_type = IStateServices.service_type; - public static final String submit_error = IStateServices.submit_error; - public static final String user = IStateServices.user; + public static final String autostart = IStateServices.SvcProps.autostart.pname(); + public static final String reference = IStateServices.SvcProps.reference.pname(); + public static final String enabled = IStateServices.SvcProps.enabled.pname(); + public static final String disable_reason = IStateServices.SvcProps.disable_reason.pname(); + public static final String endpoint = IStateServices.SvcProps.endpoint.pname(); + public static final String implementors = IStateServices.SvcProps.implementors.pname(); + public static final String instances = IStateServices.SvcProps.instances.pname(); + public static final String numeric_id = IStateServices.SvcProps.numeric_id.pname(); + public static final String ping_active = IStateServices.SvcProps.ping_active.pname(); + public static final String ping_only = IStateServices.SvcProps.ping_only.pname(); + public static final String service_alive = IStateServices.SvcProps.service_alive.pname(); + public static final String service_class = IStateServices.SvcProps.service_class.pname(); + public static final String service_dependency = IStateServices.SvcProps.service_dependency.pname(); + public static final String service_healthy = IStateServices.SvcProps.service_healthy.pname(); + public static final String service_state = IStateServices.SvcProps.service_state.pname(); + public static final String last_use = IStateServices.SvcProps.last_use.pname(); + public static final String service_statistics = IStateServices.SvcProps.service_statistics.pname(); + public static final String service_type = IStateServices.SvcProps.service_type.pname(); + public static final String submit_error = IStateServices.SvcProps.submit_error.pname(); + public static final String user = IStateServices.SvcProps.user.pname(); - public static final String service_type_CUSTOM = "CUSTOM"; + public static final String service_type_CUSTOM = IStateServices.SvcProps.CUSTOM.pname(); public static final String constant_Available = "Available"; public static final String constant_true = "true"; @@ -60,9 +60,9 @@ public class IServicesRegistry { public static final String constant_OK = "OK"; // svc - public static final String description = IStateServices.description; - public static final String process_memory_size = IStateServices.process_memory_size; - public static final String scheduling_class = IStateServices.scheduling_class; - public static final String log_directory = IStateServices.log_directory; + public static final String description = IStateServices.SvcProps.description.pname(); + public static final String process_memory_size = IStateServices.SvcProps.process_memory_size.pname(); + public static final String scheduling_class = IStateServices.SvcProps.scheduling_class.pname(); + public static final String log_directory = IStateServices.SvcProps.log_directory.pname(); } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java Tue Sep 15 14:31:24 2015 @@ -18,7 +18,6 @@ */ package org.apache.uima.ducc.ws.registry; -import java.io.IOException; import java.util.ArrayList; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; @@ -75,10 +74,10 @@ public class ServicesRegistry { String location = "refreshCache"; try { ServicesRegistryMap mapRevised = new ServicesRegistryMap(); - IStateServices iss = StateServicesFactory.getInstance(); + IStateServices iss = StateServicesFactory.getInstance(this.getClass().getName(), "WS"); StateServicesDirectory ssd = iss.getStateServicesDirectory(); if(!ssd.getDescendingKeySet().isEmpty()) { - for(Integer key : ssd.getDescendingKeySet()) { + for(Long key : ssd.getDescendingKeySet()) { StateServicesSet entry = ssd.get(key); Properties propertiesSvc = entry.get(IServicesRegistry.svc); Properties propertiesMeta = entry.get(IServicesRegistry.meta); @@ -92,7 +91,7 @@ public class ServicesRegistry { logger.debug(location, jobid, "size: "+map.size()); ServicesSortCache.getInstance().update(map); } - catch(IOException e) { + catch(Exception e) { logger.error(location, jobid, e); } } @@ -149,7 +148,7 @@ public class ServicesRegistry { try { logger.debug(location, jobid, "size: "+map.size()); logger.debug(location, jobid, "search: "+name); - for(Integer key : map.keySet()) { + for(Long key : map.keySet()) { ServicesRegistryMapPayload payload = map.get(key); Properties meta = payload.meta; if(meta != null) { @@ -176,7 +175,7 @@ public class ServicesRegistry { public String findServiceUser(String id) { String retVal = null; try { - for(Integer key : map.keySet()) { + for(Long key : map.keySet()) { ServicesRegistryMapPayload payload = map.get(key); Properties meta = payload.meta; if(meta != null) { @@ -200,7 +199,7 @@ public class ServicesRegistry { String retVal = null; try { long id = duccId.getFriendly(); - for(Integer key : map.keySet()) { + for(Long key : map.keySet()) { ServicesRegistryMapPayload payload = map.get(key); Properties meta = payload.meta; if(meta != null) { Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java Tue Sep 15 14:31:24 2015 @@ -23,15 +23,15 @@ import java.util.NavigableSet; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -public class ServicesRegistryMap extends ConcurrentHashMap<Integer,ServicesRegistryMapPayload> { +public class ServicesRegistryMap extends ConcurrentHashMap<Long,ServicesRegistryMapPayload> { private static final long serialVersionUID = 1L; - public NavigableSet<Integer> getDescendingKeySet() { - TreeMap<Integer,ServicesRegistryMapPayload> tMap = new TreeMap<Integer,ServicesRegistryMapPayload>(); - Iterator<java.util.Map.Entry<Integer, ServicesRegistryMapPayload>> iterator = entrySet().iterator(); + public NavigableSet<Long> getDescendingKeySet() { + TreeMap<Long,ServicesRegistryMapPayload> tMap = new TreeMap<Long,ServicesRegistryMapPayload>(); + Iterator<java.util.Map.Entry<Long, ServicesRegistryMapPayload>> iterator = entrySet().iterator(); while(iterator.hasNext()) { - java.util.Map.Entry<Integer, ServicesRegistryMapPayload> entry = iterator.next(); + java.util.Map.Entry<Long, ServicesRegistryMapPayload> entry = iterator.next(); tMap.put(entry.getKey(), entry.getValue()); } return tMap.descendingKeySet(); Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java?rev=1703203&r1=1703202&r2=1703203&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java Tue Sep 15 14:31:24 2015 @@ -46,7 +46,7 @@ public class ServicesSortCache { public void update(ServicesRegistryMap registryMap) { TreeMap<SortableService,IServiceAdapter> mapRevised = new TreeMap<SortableService,IServiceAdapter>(); - for(Entry<Integer, ServicesRegistryMapPayload> entry : registryMap.entrySet()) { + for(Entry<Long, ServicesRegistryMapPayload> entry : registryMap.entrySet()) { ServicesRegistryMapPayload payload = entry.getValue(); Properties meta = payload.meta; Properties svc = payload.svc; @@ -62,11 +62,11 @@ public class ServicesSortCache { for(Entry<SortableService, IServiceAdapter> entry : map.entrySet()) { IServiceAdapter payload = entry.getValue(); Properties meta = payload.getMeta(); - String key = IStateServices.numeric_id; + String key = IStateServices.SvcProps.numeric_id.pname(); String value = meta.getProperty(key); int numeric_id = Integer.parseInt(value); if(numeric_id == id) { - meta.setProperty(IStateServices.enabled, Boolean.toString(bool)); + meta.setProperty(IStateServices.SvcProps.enabled.pname(), Boolean.toString(bool)); payload.setMeta(meta); break; }
