Fix for OODT-526: Add base functionality in Curator for Task List persistence (mjoyce, mattmann)
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/208dde7c Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/208dde7c Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/208dde7c Branch: refs/heads/feature/zookeeper-config Commit: 208dde7c1c1ccc613614660d8bb4577e411e8c6b Parents: 2ff8689 Author: Chris Mattmann <[email protected]> Authored: Mon Jul 17 14:39:07 2017 -0700 Committer: Chris Mattmann <[email protected]> Committed: Mon Jul 17 14:39:07 2017 -0700 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../cas/curation/service/IngestionResource.java | 146 ++++++++++++++++++- .../org/apache/oodt/cas/curation/HomePage.html | 2 +- .../apache/oodt/cas/curation/HomePage_edrn.html | 2 +- .../curation/workbench/Workbench.js | 4 + 5 files changed, 149 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index d4cb042..081d501 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,8 @@ Apache OODT Change Log Release 1.1 - Current Development +* OODT-526 Add base functionality in Curator for Task List persistence (mjoyce, mattmann) + * OODT-525 Export IngestTaskList from Curator (mjoyce, mattmann) * OODT-304 Add a Getter method to get the file handler configuration (Michael Cayanan via mattmann) http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java ---------------------------------------------------------------------- diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java index 154ddb3..fce9ec1 100644 --- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java +++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java @@ -17,6 +17,7 @@ package org.apache.oodt.cas.curation.service; +import org.apache.oodt.cas.curation.structs.ExtractorConfig; //OODT imports import org.apache.oodt.cas.curation.structs.IngestionTask; import org.apache.oodt.cas.curation.util.DateUtils; @@ -33,12 +34,15 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.concurrent.ConcurrentHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; import java.util.Vector; @@ -50,7 +54,9 @@ import javax.xml.parsers.ParserConfigurationException; import org.w3c.dom.Document; import org.w3c.dom.Element; - +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; import org.apache.xml.serialize.OutputFormat; import org.apache.xml.serialize.XMLSerializer; @@ -89,7 +95,7 @@ public class IngestionResource extends CurationService { private static final String RESP_SUCCESS = "success"; private IngestionTaskList taskList; - + private String taskListSaveLocPath; public IngestionResource() { @@ -130,7 +136,16 @@ public class IngestionResource extends CurationService { newTask.setPolicy(policy); newTask.setProductType(productType); newTask.setStatus(IngestionTask.NOT_STARTED); - return this.taskList.addIngestionTask(newTask); + String addedTaskID = this.taskList.addIngestionTask(newTask); + saveTaskListState(); + return addedTaskID; + } + + @GET + @Path("init") + @Produces("text/plain") + public void initTaskList() { + loadSavedTaskListState(); } @GET @@ -138,6 +153,7 @@ public class IngestionResource extends CurationService { @Produces("text/plain") public void removeTask(@QueryParam("taskId") String ingestTaskId) { this.taskList.removeIngestionTask(ingestTaskId); + saveTaskListState(); } @GET @@ -195,7 +211,10 @@ public class IngestionResource extends CurationService { task.setStatus(IngestionTask.FINISHED); } - return this.encodeIngestResponseAsHTML(true, null); + String response = this.encodeIngestResponseAsHTML(true, null); + saveTaskListState(); + return response; + } private String encodeTaskListAsHTML(List<IngestionTask> taskList) { @@ -320,6 +339,113 @@ public class IngestionResource extends CurationService { } } + private void saveTaskListState() { + this.taskList.exportTaskListAsXMLToFile(this.taskListSaveLocPath); + } + + private void loadSavedTaskListState() { + File state = new File(this.taskListSaveLocPath); + if (state.exists()) + parseXMLState(state); + } + + private void parseXMLState(File stateFile) { + try { + Document document = buildXMLDocument(stateFile); + rebuildTaskList(document); + } catch (Exception e) { + LOG.log(Level.WARNING, + "parseXMLState: Unable to process saved TaskList state"); + } + } + + private Document buildXMLDocument(File stateFile) + throws ParserConfigurationException, SAXException, IOException { + + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory + .newInstance(); + DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder(); + Document document = docBuilder.parse(stateFile); + document.getDocumentElement().normalize(); + return document; + } + + private void rebuildTaskList(Document document) + throws FileNotFoundException, IOException, ParseException { + + NodeList tasks = document.getElementsByTagName("Task"); + + for (int i = 0; i < tasks.getLength(); i++) { + Element taskElement = (Element) tasks.item(i); + + String id = getNodeValue(taskElement, "ID"); + String dateString = getNodeValue(taskElement, "CreateDate"); + List<String> newFileList = getTaskFileList(taskElement); + String policy = getNodeValue(taskElement, "Policy"); + String productType = getNodeValue(taskElement, "ProductType"); + String status = getNodeValue(taskElement, "Status"); + String[] extConfParams = getTaskExtractorConfigParams(taskElement); + + IngestionTask newTask = createTaskFromParameters(id, dateString, + newFileList, policy, productType, status, extConfParams); + + this.taskList.addIngestionTaskWithoutIdGen(newTask); + } + } + + private List<String> getTaskFileList(Element parent) { + NodeList files = parent.getElementsByTagName("File"); + List<String> newFileList = new Vector<String>(); + + for (int i = 0; i < files.getLength(); i++) { + Element fileNode = (Element) files.item(i); + newFileList.add(fileNode.getFirstChild().getNodeValue()); + } + + return newFileList; + } + + private String[] getTaskExtractorConfigParams(Element parent) { + String[] params = new String[2]; + + NodeList extData = parent.getElementsByTagName("ExtractorConfig"); + Element extDataElm = (Element) extData.item(0); + params[0] = extDataElm.getElementsByTagName("UploadPath").item(0) + .getFirstChild().getNodeValue(); + params[1] = extDataElm.getElementsByTagName("ID").item(0).getFirstChild() + .getNodeValue(); + + return params; + } + + private IngestionTask createTaskFromParameters(String id, String dateString, + List<String> fileList, String policy, String productType, String status, + final String[] extConfParams) + throws FileNotFoundException, IOException, ParseException { + + Date date = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy", + Locale.ENGLISH).parse(dateString); + ExtractorConfig extConf = ExtractorConfigReader + .readFromDirectory(new File(extConfParams[0]), extConfParams[1]); + + IngestionTask newTask = new IngestionTask(); + newTask.setId(id); + newTask.setCreateDate(date); + newTask.setFileList(fileList); + newTask.setPolicy(policy); + newTask.setProductType(productType); + newTask.setStatus(status); + newTask.setExtConf(extConf); + + return newTask; + } + + private String getNodeValue(Element element, String tag) { + NodeList nodeList = element.getElementsByTagName(tag).item(0) + .getChildNodes(); + return ((Node) nodeList.item(0)).getNodeValue(); + } + class IngestionTaskList { private ConcurrentHashMap<String, IngestionTask> taskMap; @@ -328,6 +454,10 @@ public class IngestionResource extends CurationService { this.taskMap = new ConcurrentHashMap<String, IngestionTask>(); } + public int getNumberOfTasks() { + return taskMap.size(); + } + public synchronized String addIngestionTask(IngestionTask task) { this.provideTaskId(task); taskMap.put(task.getId(), task); @@ -338,6 +468,12 @@ public class IngestionResource extends CurationService { taskMap.remove(taskId); } + public synchronized String addIngestionTaskWithoutIdGen( + IngestionTask task) { + taskMap.put(task.getId(), task); + return task.getId(); + } + public IngestionTask getIngestionTaskById(String taskId) { return taskMap.get(taskId); } @@ -360,7 +496,7 @@ public class IngestionResource extends CurationService { return taskList; } - //TODO: write method to load ingest task list from XML file + // TODO: write method to load ingest task list from XML file public void exportTaskListAsXMLToFile(String fileName) { try { Document xmlDocument = generateXMLDocument(); http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html ---------------------------------------------------------------------- diff --git a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html index a0d8911..cba9178 100644 --- a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html +++ b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html @@ -27,7 +27,7 @@ </wicket:remove> </head> -<body> +<body onload="initIngestTaskList();refreshIngestTaskList();registerIngestionTaskListener();"> <div id="page"> <div id="cas-header"> <div id="cas-logo-left"><!-- cas logo lhs --></div> http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html ---------------------------------------------------------------------- diff --git a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html index e017835..a1f7875 100644 --- a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html +++ b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html @@ -20,7 +20,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> </head> -<body> +<body onload="initIngestTaskList();refreshIngestTaskList();registerIngestionTaskListener();"> <div id="page"> <div id="ncibanner"> http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js ---------------------------------------------------------------------- diff --git a/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js b/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js index 3d0601f..65f658e 100644 --- a/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js +++ b/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js @@ -320,6 +320,10 @@ function createIngestionTaskRow(data) { $('#ingestionTaskListItems > tbody').replace(data); } +function initIngestTaskList() { + $.get('./services/ingest/init'); +} + function removeIngestionTask(id) { $.get('./services/ingest/remove', { 'taskId' : id },
