Author: degenaro
Date: Tue Oct 18 14:56:01 2016
New Revision: 1765450
URL: http://svn.apache.org/viewvc?rev=1765450&view=rev
Log:
UIMA-5060 DUCC Orchestrator (OR) "warm" restart issues
- for restoration of next job/reservation sequence number, use the greater of
seqno in state/orchestrator.properties and historical/ckpt from database
comprising jobs+reservations
- log a WARNing in or.log if historical/ckpt data is used
Added:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
(with props)
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/IPropertiesFileManager.java
Tue Oct 18 14:56:01 2016
@@ -20,4 +20,6 @@ package org.apache.uima.ducc.common.pers
public interface IPropertiesFileManager {
public int increment(String key);
+ public String get(String key, String defaultValue);
+ public void set(String key, String value);
}
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
Tue Oct 18 14:56:01 2016
@@ -62,4 +62,32 @@ public class DuccIdFactory implements ID
return new DuccId(seqno.get());
}
}
+
+ /**
+ * set seqno to candidate if it is bigger than previous, return
previous value
+ */
+ public long setIfMax(long candidate) {
+ long previous = seqno.get();
+ synchronized(this) {
+ if(propertiesFileManager != null) {
+ try {
+ String propertiesFileValue =
propertiesFileManager.get(propertiesFileKey, ""+previous);
+ previous =
Long.parseLong(propertiesFileValue);
+ }
+ catch(Exception e) {
+ // No worries
+ }
+ if(candidate > previous) {
+
propertiesFileManager.set(propertiesFileKey, ""+candidate);
+ seqno.set(candidate);
+ }
+ }
+ else {
+ if(candidate > previous) {
+ seqno.set(candidate);
+ }
+ }
+ }
+ return previous;
+ }
}
Modified:
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/IDuccIdFactory.java
Tue Oct 18 14:56:01 2016
@@ -21,5 +21,5 @@ package org.apache.uima.ducc.common.util
public interface IDuccIdFactory {
public DuccId next();
-
+ public long setIfMax(long s);
}
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java?rev=1765450&r1=1765449&r2=1765450&view=diff
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
(original)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCommonArea.java
Tue Oct 18 14:56:01 2016
@@ -46,6 +46,7 @@ public class OrchestratorCommonArea {
private static OrchestratorCommonArea orchestratorCommonArea = null;
private static final DuccLogger logger =
DuccLoggerComponents.getOrLogger(OrchestratorCommonArea.class.getName());
+ private static final DuccId jobid = null;
private static DuccPropertiesResolver dpr =
DuccPropertiesResolver.getInstance();
@@ -118,7 +119,8 @@ public class OrchestratorCommonArea {
ComponentHelper.oneInstance(IDuccEnv.DUCC_STATE_DIR,"orchestrator");
}
// </Jira 3414>
- setPropertiesFileManager(new
PropertiesFileManager(IDuccLoggerComponents.abbrv_orchestrator,
IDuccEnv.DUCC_STATE_DIR, constOrchestratorProperties, false, true));
+ PropertiesFileManager pfm = new
PropertiesFileManager(IDuccLoggerComponents.abbrv_orchestrator,
IDuccEnv.DUCC_STATE_DIR, constOrchestratorProperties, false, true);
+ setPropertiesFileManager(pfm);
initSeqNo();
setDuccIdFactory(new
DuccIdFactory(propertiesFileManager,constSeqNo));
workMap = new DuccWorkMap();
@@ -134,6 +136,13 @@ public class OrchestratorCommonArea {
System.exit(1); // what should we do here? exit or acquire
the NullHistoryManager?
}
logger.info(methodName, null, "Got history manager of class",
historyPersistenceManager.getClass().getName());
+ //
+ OrchestratorRecovery orchestratorRecovery = new
OrchestratorRecovery(historyPersistenceManager);
+ long historicSeqNo = orchestratorRecovery.recoverSeqNo();
+ long previousSeqNo = getDuccIdFactory().setIfMax(historicSeqNo);
+ if(previousSeqNo != historicSeqNo) {
+ logger.warn(methodName, jobid, "properties:"+previousSeqNo+"
"+""+"historic:"+historicSeqNo);
+ }
}
public String getStateDirectory() {
Added:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
URL:
http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java?rev=1765450&view=auto
==============================================================================
---
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
(added)
+++
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
Tue Oct 18 14:56:01 2016
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.orchestrator;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import
org.apache.uima.ducc.transport.event.common.history.IHistoryPersistenceManager;
+
+/**
+ * Purpose: scan job+reservation history+checkpoint data for the largest
sequence number
+ */
+public class OrchestratorRecovery {
+
+ private static final DuccLogger logger =
DuccLoggerComponents.getOrLogger(OrchestratorRecovery.class.getName());
+ private static final DuccId jobid = null;
+
+ IHistoryPersistenceManager historyPersistenceManager = null;
+
+ public OrchestratorRecovery(IHistoryPersistenceManager
historyPersistenceManager) {
+ setHistoryPersistenceManager(historyPersistenceManager);
+ }
+
+ private void setHistoryPersistenceManager(IHistoryPersistenceManager
value) {
+ historyPersistenceManager = value;
+ }
+
+ private Long findHistoryLastJob() {
+ String location = "findHistoryLastJob";
+ Long retVal = new Long(-1);
+ try {
+ List<IDuccWorkJob> duccWorkJobs =
historyPersistenceManager.restoreJobs(1);
+ if(duccWorkJobs != null) {
+ if(!duccWorkJobs.isEmpty()) {
+ for(IDuccWorkJob job : duccWorkJobs) {
+ DuccId duccId = job.getDuccId();
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid,
"hist job "+retVal);
+ break;
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist job
empty");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist job not
found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private Long findHistoryLastReservation() {
+ String location = "findHistoryLastReservation";
+ Long retVal = new Long(-1);
+ try {
+ List<IDuccWorkReservation> duccWorkReservations =
historyPersistenceManager.restoreReservations(1);
+ if(duccWorkReservations != null) {
+ if(!duccWorkReservations.isEmpty()) {
+ for(IDuccWorkReservation reservation :
duccWorkReservations) {
+ DuccId duccId =
reservation.getDuccId();
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid,
"hist reservation "+retVal);
+ break;
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist
reservation empty");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "hist reservation
not found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private Long findCheckpointLastJob() {
+ String location = "findCheckpointLastJob";
+ Long retVal = new Long(-1);
+ DuccWorkMap workMap =
OrchestratorCommonArea.getInstance().getWorkMap();
+ try {
+ if(workMap != null) {
+ DuccId duccId = getMax(workMap.getJobKeySet());
+ if(duccId != null) {
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid, "ckpt job
"+retVal);
+ }
+ else {
+ logger.debug(location, jobid, "ckpt job
not found");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "workmap not
found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private Long findCheckpointLastReservation() {
+ String location = "findCheckpointLastReservation";
+ Long retVal = new Long(-1);
+ DuccWorkMap workMap =
OrchestratorCommonArea.getInstance().getWorkMap();
+ try {
+ if(workMap != null) {
+ DuccId duccId =
getMax(workMap.getReservationKeySet());
+ if(duccId != null) {
+ retVal = duccId.getFriendly();
+ logger.info(location, jobid, "ckpt
reservation "+retVal);
+ }
+ else {
+ logger.debug(location, jobid, "ckpt
reservation not found");
+ }
+ }
+ else {
+ logger.debug(location, jobid, "workmap not
found");
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, jobid, e);
+ }
+ return retVal;
+ }
+
+ private DuccId getMax(Set<DuccId> keys) {
+ DuccId retVal = null;
+ if(keys != null) {
+ for(DuccId key : keys) {
+ if(retVal == null) {
+ retVal = key;
+ }
+ else if(key.getFriendly() >
retVal.getFriendly()) {
+ retVal = key;
+ }
+ }
+ }
+ return retVal;
+ }
+
+ private Long maxOf(Long v0, Long v1, String text) {
+ String location = "maxOf";
+ Long retVal = v0;
+ if(v1 > v0) {
+ retVal = v1+1;
+ logger.info(location, jobid, text);
+ }
+ return retVal;
+ }
+
+ public Long recoverSeqNo() {
+ String location = "recoverSeqNo";
+ Long seqNo = new Long(-1);
+ seqNo = maxOf(seqNo, findHistoryLastJob(), "history:job");
+ seqNo = maxOf(seqNo, findHistoryLastReservation(),
"history:reservation");
+ seqNo = maxOf(seqNo, findCheckpointLastJob(), "checkpoint:job");
+ seqNo = maxOf(seqNo, findCheckpointLastReservation(),
"checkpoint:reservation");
+ logger.info(location, jobid, seqNo);
+ return seqNo;
+ }
+}
Propchange:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorRecovery.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
