Author: challngr Date: Wed Apr 24 17:38:35 2013 New Revision: 1471555 URL: http://svn.apache.org/r1471555 Log: UIMA-2833 Save service registrations and associated items to history.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java?rev=1471555&r1=1471554&r2=1471555&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceHandler.java Wed Apr 24 17:38:35 2013 @@ -1089,19 +1089,6 @@ public class ServiceHandler } sset.deleteProperties(); - - // String metafn = sset.getMetaFilename(); - // String propsfn = sset.getPropsFilename(); - - // if ( metafn != null ) { - // File mf = new File(metafn); - // mf.delete(); - // } - // if ( propsfn != null ) { - // File pf = new File(propsfn); - // pf.delete(); - // } - } String extractId(long friendly, String epname) Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1471555&r1=1471554&r2=1471555&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java Wed Apr 24 17:38:35 2013 @@ -133,6 +133,10 @@ public class ServiceManagerComponent if ( ! descdir.exists() ) { descdir.mkdirs(); } + File histdir = new File(serviceHistoryLocation()); + if ( ! histdir.exists() ) { + histdir.mkdirs(); + } String[] desclist = descdir.list(); for ( String d : desclist) { if ( d.endsWith(".svc") ) { @@ -151,18 +155,7 @@ public class ServiceManagerComponent String sc = metaprops.getProperty("service-class"); if ( (sc != null) && ( sc.equals("Implicit") || sc.equals("Submitted") ) ) { logger.info(methodName, null, "Scrubbing", sc, "service", stem); - try { - File mf = new File(meta_filename); - mf.delete(); - } catch ( Throwable t ) { - // nothing to do about it, ignore. - } - try { - File pf = new File(props_filename); - pf.delete(); - } catch ( Throwable t ) { - // nothing to do about it, ignore. - } + deleteProperties(stem, meta_filename, metaprops, props_filename, props); continue; } @@ -566,11 +559,16 @@ public class ServiceManagerComponent notify(); } - private String serviceFileLocation() + static String serviceFileLocation() { return System.getProperty("DUCC_HOME") + "/state/services"; } + static String serviceHistoryLocation() + { + return System.getProperty("DUCC_HOME") + "/history/services-registry/"; + } + private String serviceFileKey(String fn) { return serviceFileLocation() + "/" + fn; @@ -722,14 +720,58 @@ public class ServiceManagerComponent //ev.setReply(ServiceCode.OK, "Service not implemented.", "no-endpoint", null); } - public synchronized DuccId newId() + Object idSync = new Object(); + public DuccId newId() throws Exception { - DuccId id = idFactory.next(); - sm_props.setProperty(service_seqno, id.toString()); - FileOutputStream fos = new FileOutputStream(state_file); - sm_props.store(fos, "Service Manager Properties"); - fos.close(); + DuccId id = null; + synchronized(idSync) { + id = idFactory.next(); + sm_props.setProperty(service_seqno, id.toString()); + FileOutputStream fos = new FileOutputStream(state_file); + sm_props.store(fos, "Service Manager Properties"); + fos.close(); + } return id; } + + + static void deleteProperties(String id, String meta_filename, Properties meta_props, String props_filename, Properties job_props) + { + // NOTE: During init we may now know the ID as a DuccId so it has to be passed in as a string + + String methodName = "deleteProperties"; + // Save a copy in history, and then delete the original + String history_dir = serviceHistoryLocation(); + if ( meta_filename != null ) { + File mfh = new File(history_dir + id + ".meta"); + try { + FileOutputStream fos = new FileOutputStream(mfh); + meta_props.store(fos, "Archived meta descriptor"); + fos.close(); + } catch (Exception e) { + logger.warn(methodName, null, id + ": Unable to save history to \"" + mfh.toString(), ": ", e.toString() + "\""); + } + + File mf = new File(meta_filename); + mf.delete(); + } + meta_filename = null; + + if ( props_filename != null ) { + File pfh = new File(history_dir + id + ".svc"); + try { + FileOutputStream fos = new FileOutputStream(pfh); + job_props.store(fos, "Archived meta descriptor"); + fos.close(); + } catch (Exception e) { + logger.warn(methodName, null, id + ":Unable to save history to \"" + pfh.toString(), ": ", e.toString() + "\""); + } + + File pf = new File(props_filename); + pf.delete(); + } + props_filename = null; + } + } Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1471555&r1=1471554&r2=1471555&view=diff ============================================================================== --- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java (original) +++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Wed Apr 24 17:38:35 2013 @@ -19,7 +19,6 @@ package org.apache.uima.ducc.sm; import java.io.BufferedReader; -import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -75,6 +74,7 @@ public class ServiceSet // For a registered service, here is my registered id DuccId id; HashMap<Long, DuccId> friendly_ids = new HashMap<Long, DuccId>(); + String history_key = "work-instances"; // incoming nodes, for dup checking List<ServiceSet> predecessors = new ArrayList<ServiceSet>(); @@ -312,17 +312,18 @@ public class ServiceSet synchronized void deleteProperties() { - if ( meta_filename != null ) { - File mf = new File(meta_filename); - mf.delete(); - } - meta_filename = null; - if ( props_filename != null ) { - File pf = new File(props_filename); - pf.delete(); - } - props_filename = null; + // be sure to move any services that seem not to have croaked yet to history + String history = meta_props.getStringProperty(history_key, ""); + for ( Long id : friendly_ids.keySet() ) { + history = history + " " + id.toString(); + } + meta_props.put(history_key, history); + meta_props.remove("implementors"); + + ServiceManagerComponent.deleteProperties(id.toString(), meta_filename, meta_props, props_filename, job_props); + meta_filename = null; + props_filename = null; } void setIncoming(ServiceSet sset) @@ -422,6 +423,7 @@ public class ServiceSet void synchronizeImplementors(Map<DuccId, JobState> work) { HashMap<Long, DuccId> newmap = new HashMap<Long, DuccId>(); + // first loop synchronized 'friendly_ids' with live jobs comining in for ( DuccId id : work.keySet() ) { long fid = id.getFriendly(); @@ -432,6 +434,16 @@ public class ServiceSet } } + // second loop synchronizes history with jobs that used to be live and aren't any more (because of restart) + String history = meta_props.getStringProperty(history_key, ""); + for ( Long friendly : friendly_ids.keySet() ) { + DuccId id = newmap.get(friendly); + if ( id == null ) { + history = history + " " + friendly; + } + } + meta_props.put(history_key, history); + friendly_ids = newmap; // replace persisted version with validated version from OR state persistImplementors(); } @@ -711,10 +723,15 @@ public class ServiceSet public void removeImplementor(DuccId id) { + String methodName = "removeImplementors"; if ( ! implementors.containsKey(id ) ) return; // quick short circuit if it's already gone + logger.debug(methodName, this.id, "Removing implementor", id); implementors.remove(id); friendly_ids.remove(id.getFriendly()); + String history = meta_props.getStringProperty(history_key, ""); + history = history + " " + id.toString(); + meta_props.put(history_key, history); persistImplementors(); if ( implementors.size() == 0 ) { stopPingThread();