Author: kwright
Date: Mon Jun 2 12:43:35 2014
New Revision: 1599193
URL: http://svn.apache.org/r1599193
Log:
Revise pipeline stage portion of JobDescription to make more sense
Modified:
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
Modified:
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java?rev=1599193&r1=1599192&r2=1599193&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
(original)
+++
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/interfaces/IJobDescription.java
Mon Jun 2 12:43:35 2014
@@ -96,26 +96,40 @@ public interface IJobDescription
public void clearPipeline();
/** Add a pipeline connection.
- *@param pipelineConnectionName is the name of the pipeline connection to add.
+ *@param pipelineStageConnectionName is the name of the pipeline connection
to add.
+ *@param pipelineStageDescription is a description of the pipeline stage
being added.
+ *@return the empty output specification for this pipeline stage.
*/
- public void addPipelineConnection(String pipelineConnectionName);
+ public OutputSpecification addPipelineStage(String
pipelineStageConnectionName, String pipelineStageDescription);
/** Get a count of pipeline connections.
*@return the current number of pipeline connections.
*/
- public int countPipelineConnections();
+ public int countPipelineStages();
/** Get a specific pipeline connection name.
- *@param index is the index of the pipeline whose connection name to get.
+ *@param index is the index of the pipeline stage whose connection name to
get.
*@return the name of the connection.
*/
- public String getPipelineConnectionName(int index);
-
- /** Get a specific pipeline connection specification.
- *@param index is the index of the pipeline whose specification is needed.
+ public String getPipelineStageConnectionName(int index);
+
+ /** Get a specific pipeline stage description.
+ *@param index is the index of the pipeline stage whose description to get.
+ *@return the name of the connection.
+ */
+ public String getPipelineStageDescription(int index);
+
+ /** Get a specific pipeline stage specification.
+ *@param index is the index of the pipeline stage whose specification is
needed.
*@return the specification for the connection.
*/
- public OutputSpecification getPipelineSpecification(int index);
+ public OutputSpecification getPipelineStageSpecification(int index);
+
+ /** Delete a pipeline stage.
+ *@param index is the index of the pipeline stage to delete.
+ */
+ public void deletePipelineStage(int index);
+
/** Set the job type.
*@param type is the type (as an integer).
Modified:
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java?rev=1599193&r1=1599192&r2=1599193&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
(original)
+++
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/JobDescription.java
Mon Jun 2 12:43:35 2014
@@ -44,7 +44,7 @@ public class JobDescription implements I
protected String description = null;
protected String outputConnectionName = null;
protected String connectionName = null;
- protected final List<String> pipelineConnectionNames = new
ArrayList<String>();
+ protected final List<PipelineStage> pipelineStages = new
ArrayList<PipelineStage>();
protected int type = TYPE_CONTINUOUS;
protected int startMethod = START_WINDOWBEGIN;
protected int priority = 5;
@@ -73,9 +73,6 @@ public class JobDescription implements I
// Document specification
protected DocumentSpecification documentSpecification = new
DocumentSpecification();
- // Transformation specifications
- protected final List<OutputSpecification> pipelineSpecifications = new
ArrayList<OutputSpecification>();
-
// Hop count filters.
protected HashMap hopCountFilters = new HashMap();
@@ -100,10 +97,14 @@ public class JobDescription implements I
rval.id = id;
rval.isNew = isNew;
rval.outputConnectionName = outputConnectionName;
+ // Direct modification of this object is possible - so it also has to know
if it is read-only!!
+ rval.outputSpecification = outputSpecification.duplicate(readOnly);
rval.connectionName = connectionName;
- for (String pipelineConnectionName : pipelineConnectionNames)
+ // Direct modification of this object is possible - so it also has to know
if it is read-only!!
+ rval.documentSpecification = documentSpecification.duplicate(readOnly);
+ for (PipelineStage pipelineStage : pipelineStages)
{
- rval.pipelineConnectionNames.add(pipelineConnectionName);
+ rval.pipelineStages.add(new
PipelineStage(pipelineStage.getConnectionName(),pipelineStage.getDescription(),pipelineStage.getSpecification().duplicate(readOnly)));
}
rval.description = description;
rval.type = type;
@@ -132,15 +133,6 @@ public class JobDescription implements I
rval.addForcedMetadataValue(forcedParamName,value);
}
}
- // Direct modification of this object is possible - so it also has to know
if it is read-only!!
- rval.outputSpecification = outputSpecification.duplicate(readOnly);
- // Direct modification of this object is possible - so it also has to know
if it is read-only!!
- rval.documentSpecification = documentSpecification.duplicate(readOnly);
- // Duplicate the pipeline specifications
- for (OutputSpecification pipelineSpecification : pipelineSpecifications)
- {
-
rval.pipelineSpecifications.add(pipelineSpecification.duplicate(readOnly));
- }
rval.readOnly = readOnly;
return rval;
}
@@ -261,41 +253,70 @@ public class JobDescription implements I
{
if (readOnly)
throw new IllegalStateException("Attempt to change read-only object");
- pipelineConnectionNames.clear();
- pipelineSpecifications.clear();
+ pipelineStages.clear();
}
- /** Add a pipeline connection */
+ /** Add a pipeline connection.
+ *@param pipelineStageConnectionName is the name of the pipeline connection
to add.
+ *@param pipelineStageDescription is a description of the pipeline stage
being added.
+ *@return the empty output specification for this pipeline stage.
+ */
@Override
- public void addPipelineConnection(String pipelineConnectionName)
+ public OutputSpecification addPipelineStage(String
pipelineStageConnectionName, String pipelineStageDescription)
{
if (readOnly)
throw new IllegalStateException("Attempt to change read-only object");
- pipelineConnectionNames.add(pipelineConnectionName);
- pipelineSpecifications.add(new OutputSpecification());
+ PipelineStage ps = new
PipelineStage(pipelineStageConnectionName,pipelineStageDescription);
+ pipelineStages.add(ps);
+ return ps.getSpecification();
}
- /** Get a count of pipeline connections */
+ /** Get a count of pipeline stages */
@Override
- public int countPipelineConnections()
+ public int countPipelineStages()
{
- return pipelineConnectionNames.size();
+ return pipelineStages.size();
}
- /** Get a specific pipeline connection name */
+ /** Get a specific pipeline connection name.
+ *@param index is the index of the pipeline stage whose connection name to
get.
+ *@return the name of the connection.
+ */
@Override
- public String getPipelineConnectionName(int index)
+ public String getPipelineStageConnectionName(int index)
{
- return pipelineConnectionNames.get(index);
+ return pipelineStages.get(index).getConnectionName();
}
- /** Get a specific pipeline connection specification */
+ /** Get a specific pipeline stage description.
+ *@param index is the index of the pipeline stage whose description to get.
+ *@return the name of the connection.
+ */
@Override
- public OutputSpecification getPipelineSpecification(int index)
+ public String getPipelineStageDescription(int index)
{
- return pipelineSpecifications.get(index);
+ return pipelineStages.get(index).getDescription();
}
-
+
+ /** Get a specific pipeline stage specification.
+ *@param index is the index of the pipeline stage whose specification is
needed.
+ *@return the specification for the connection.
+ */
+ @Override
+ public OutputSpecification getPipelineStageSpecification(int index)
+ {
+ return pipelineStages.get(index).getSpecification();
+ }
+
+ /** Delete a pipeline stage.
+ *@param index is the index of the pipeline stage to delete.
+ */
+ @Override
+ public void deletePipelineStage(int index)
+ {
+ pipelineStages.remove(index);
+ }
+
/** Set the job type.
*@param type is the type (as an integer).
*/
@@ -598,4 +619,40 @@ public class JobDescription implements I
rval.add(value);
}
+ protected static class PipelineStage
+ {
+ protected final String connectionName;
+ protected final String description;
+ protected final OutputSpecification specification;
+
+ public PipelineStage(String connectionName, String description)
+ {
+ this.connectionName = connectionName;
+ this.description = description;
+ this.specification = new OutputSpecification();
+ }
+
+ public PipelineStage(String connectionName, String description,
OutputSpecification spec)
+ {
+ this.connectionName = connectionName;
+ this.description = description;
+ this.specification = spec;
+ }
+
+ public OutputSpecification getSpecification()
+ {
+ return specification;
+ }
+
+ public String getConnectionName()
+ {
+ return connectionName;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+ }
+
}
Modified:
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java?rev=1599193&r1=1599192&r2=1599193&view=diff
==============================================================================
---
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
(original)
+++
manifoldcf/branches/CONNECTORS-946/framework/pull-agent/src/main/java/org/apache/manifoldcf/crawler/jobs/PipelineManager.java
Mon Jun 2 12:43:35 2014
@@ -34,6 +34,7 @@ import java.util.*;
* <tr><td>ownerid</td><td>BIGINT</td><td>Reference:jobs.id</td></tr>
* <tr><td>ordinal</td><td>BIGINT</td><td></td></tr>
* <tr><td>transformationname</td><td>VARCHAR(32)</td><td></td></tr>
+* <tr><td>transformationdesc</td><td>VARCHAR(255)</td><td></td></tr>
* <tr><td>transformationspec</td><td>LONGTEXT</td><td></td></tr>
* </table>
* <br><br>
@@ -47,6 +48,7 @@ public class PipelineManager extends org
public final static String ownerIDField = "ownerid";
public final static String ordinalField = "ordinal";
public final static String transformationNameField = "transformationname";
+ public final static String transformationDescriptionField =
"transformationdesc";
public final static String transformationSpecField = "transformationspec";
/** Constructor.
@@ -76,6 +78,7 @@ public class PipelineManager extends org
map.put(ownerIDField,new
ColumnDescription("BIGINT",false,false,ownerTable,owningTablePrimaryKey,false));
map.put(ordinalField,new
ColumnDescription("BIGINT",false,false,null,null,false));
map.put(transformationNameField,new
ColumnDescription("VARCHAR(32)",false,false,transformationTableName,transformationNameField,false));
+ map.put(transformationDescriptionField,new
ColumnDescription("VARCHAR(255)",false,true,null,null,false));
map.put(transformationSpecField,new
ColumnDescription("LONGTEXT",false,true,null,null,false));
performCreate(map,null);
}
@@ -132,12 +135,11 @@ public class PipelineManager extends org
{
IResultRow row = set.getRow(i);
Long ownerID = (Long)row.getValue(ownerIDField);
- Long ordinal = (Long)row.getValue(ordinalField);
String transformationName =
(String)row.getValue(transformationNameField);
+ String transformationDesc =
(String)row.getValue(transformationDescriptionField);
String transformationSpec =
(String)row.getValue(transformationSpecField);
JobDescription jd = returnValues.get(ownerID);
- jd.addPipelineConnection(transformationName);
-
jd.getPipelineSpecification((int)ordinal.longValue()).fromXML(transformationSpec);
+
jd.addPipelineStage(transformationName,transformationDesc).fromXML(transformationSpec);
}
}
@@ -152,14 +154,17 @@ public class PipelineManager extends org
try
{
HashMap map = new HashMap();
- for (int i = 0; i < job.countPipelineConnections(); i++)
+ for (int i = 0; i < job.countPipelineStages(); i++)
{
- String pipelineConnectionName = job.getPipelineConnectionName(i);
- OutputSpecification os = job.getPipelineSpecification(i);
+ String pipelineConnectionName = job.getPipelineStageConnectionName(i);
+ String pipelineStageDescription = job.getPipelineStageDescription(i);
+ OutputSpecification os = job.getPipelineStageSpecification(i);
map.clear();
map.put(ownerIDField,ownerID);
map.put(ordinalField,new Long((long)i));
map.put(transformationNameField,pipelineConnectionName);
+ if (pipelineStageDescription != null &&
pipelineStageDescription.length() > 0)
+ map.put(transformationDescriptionField,pipelineStageDescription);
map.put(transformationSpecField,os.toXML());
performInsert(map,null);
i++;
