Author: kwright
Date: Fri May 29 10:04:57 2015
New Revision: 1682411
URL: http://svn.apache.org/r1682411
Log:
Pull up fix for CONNECTORS-1204 from trunk.
Modified:
manifoldcf/branches/dev_1x/ (props changed)
manifoldcf/branches/dev_1x/CHANGES.txt
manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
manifoldcf/branches/dev_1x/framework/ (props changed)
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri May 29 10:04:57 2015
@@ -51,6 +51,7 @@
/manifoldcf/branches/CONNECTORS-1177:1670213-1670613
/manifoldcf/branches/CONNECTORS-120:1406712-1407974,1407982-1411043,1411049-1416451
/manifoldcf/branches/CONNECTORS-120-1:1416450-1417056
+/manifoldcf/branches/CONNECTORS-1204:1682204-1682409
/manifoldcf/branches/CONNECTORS-13:1525862-1527182,1539324-1541634
/manifoldcf/branches/CONNECTORS-470:1349741-1360750,1360808
/manifoldcf/branches/CONNECTORS-474:1349741-1353803
@@ -123,4 +124,4 @@
/manifoldcf/branches/CONNECTORS-981:1605049-1605773
/manifoldcf/branches/CONNECTORS-989:1611600-1612101
/manifoldcf/branches/CONNECTORS-990:1610284-1610707
-/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605,1649628,1649794,1650351,1650722,1650741-1650742,1650745,1650747,1650911,1650954,1651332,1651539,1651907,1651921,1652071,1652974,1653175,1653899,1654651,1655205,1655261,1655264,1655377,1655411,1655618,1655914,1657346,1657443,1658004,1658036,1658121,1658155,1658188,1658463,1658476,
1660258,1660276,1661454,1665848,1666160,1666781,1666820,1668312,1669100,1669238,1669487,1669523,1669586,1669660,1670614,1670625,1670715,1671496,1672169,1672301,1672616,1672737,1673559,1673573,1673579,1673722,1675781,1675898,1676094,1676882,1676910,1678300,1678329,1678471,1678551,1679730,1679826,1681390,1681735,1682232,1682252
+/manifoldcf/trunk:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1631750,1631953,1632013,1632225,1632289,1632562,1632844,1632847,1632854,1633062-1633063,1633108,1633193,1633202,1633282,1633284,1633295,1633336,1633339,1633345,1633348,1633364,1633378,1633383,1633432,1633546,1633590,1633634,1633668,1633727,1633760,1633764,1633786,1633910,1633923,1634021,1634028,1634067,1634132,1634145,1634148,163
4155,1634188,1634202,1634264,1634373,1634530,1634688,1634850,1634857,1635103,1635116,1635421,1635438,1635478,1635481,1635484,1635490,1635809,1635939,1636146,1636167,1636180,1636207,1636215,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1637310,1637350,1637364,1637373,1637378,1639259,1639593,1639600,1640018,1640101,1640199,1640204,1640208,1640314,1640319,1640749,1640772,1640805,1640888,1640925,1640941-1640942,1641222,1641328,1641557,1641559,1641629,1641633,1641724,1641754,1641911,1642163,1642255,1642318,1642531,1642650,1642658,1642673,1642716,1644197,1644399,1644538,1644920,1644931,1646317,1646397,1646403,1646408,1646640,1646947,1647574,1647585,1647608,1648686,1648976,1649201,1649203,1649529,1649605,1649628,1649794,1650351,1650722,1650741-1650742,1650745,1650747,1650911,1650954,1651332,1651539,1651907,1651921,1652071,1652974,1653175,1653899,1654651,1655205,1655261,1655264,1655377,1655411,1655618,1655914,1657346,1657443,1658004,1658036,1658121,1658155,1658188,1658463,1658476,
1660258,1660276,1661454,1665848,1666160,1666781,1666820,1668312,1669100,1669238,1669487,1669523,1669586,1669660,1670614,1670625,1670715,1671496,1672169,1672301,1672616,1672737,1673559,1673573,1673579,1673722,1675781,1675898,1676094,1676882,1676910,1678300,1678329,1678471,1678551,1679730,1679826,1681390,1681735,1682232,1682252,1682410
Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Fri May 29 10:04:57 2015
@@ -3,6 +3,10 @@ $Id$
======================= 1.10-dev =====================
+CONNECTORS-1204: Add ability to post original binary length of document
+to Solr from JCIFS connector. Also improve efficiency of JCIFS connector.
+(Karl Wright)
+
CONNECTORS-1193: Add ability to discard web content based on a
set of regular expressions.
(Arcadius Ahouansou)
Modified:
manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/jcifs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/sharedrive/SharedDriveConnector.java
Fri May 29 10:04:57 2015
@@ -32,6 +32,7 @@ import org.apache.manifoldcf.crawler.sys
import java.io.*;
import java.net.MalformedURLException;
+import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.*;
@@ -447,15 +448,11 @@ public class SharedDriveConnector extend
}
}
}
- catch (java.net.SocketTimeoutException e)
+ catch (MalformedURLException e)
{
- throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
+ throw new ManifoldCFException("Could not get a canonical path:
"+e.getMessage(),e);
}
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (IOException e)
+ catch (UnknownHostException e)
{
throw new ManifoldCFException("Could not get a canonical path:
"+e.getMessage(),e);
}
@@ -546,116 +543,142 @@ public class SharedDriveConnector extend
String[] documentDeny = null;
boolean documentSecurityOn = false;
+ // Common info we really need to fetch only once
+ long fileLength = 0L;
+ long lastModified = 0L;
+ boolean fileExists = false;
+ boolean fileIsDirectory = false;
+
try
{
file = new SmbFile(documentIdentifier,pa);
+ fileExists = fileExists(file);
// File has to exist AND have a non-null canonical path to be
readable. If the canonical path is
// null, it means that the windows permissions are not right and
directory/file is not readable!!!
String newPath = getFileCanonicalPath(file);
// We MUST check the specification here, otherwise a recrawl may not
delete what it's supposed to!
- if (fileExists(file) && newPath != null &&
checkInclude(file,newPath,spec,activities))
+ if (fileExists && newPath != null)
{
- if (fileIsDirectory(file))
+ fileIsDirectory = fileIsDirectory(file);
+ if (checkInclude(fileIsDirectory,newPath,spec))
{
- // Hmm, this is not correct; version string should be empty for
windows directories, since
- // they are not hierarchical in modified date propagation.
- // It's a directory. The version ID will be the
- // last modified date.
- //long lastModified = fileLastModified(file);
- //versionString = new Long(lastModified).toString();
- versionString = "";
-
- }
- else
- {
- // It's a file of acceptable length.
- // The ability to get ACLs, list files, and an inputstream under
DFS all work now.
- // The SmbFile for parentFolder acls.
- SmbFile parentFolder = new SmbFile(file.getParent(),pa);
-
- // Compute the security information
- String[] modelArray = new String[0];
-
- List<String> allowList = new ArrayList<String>();
- List<String> denyList = new ArrayList<String>();
- shareSecurityOn = getFileShareSecuritySet(allowList, denyList,
file, shareAcls);
- shareAllow = allowList.toArray(modelArray);
- shareDeny = denyList.toArray(modelArray);
-
- allowList.clear();
- denyList.clear();
- parentSecurityOn = getFileSecuritySet(allowList, denyList,
parentFolder, parentFolderAcls);
- parentAllow = allowList.toArray(modelArray);
- parentDeny = denyList.toArray(modelArray);
-
- allowList.clear();
- denyList.clear();
- documentSecurityOn = getFileSecuritySet(allowList, denyList, file,
acls);
- documentAllow = allowList.toArray(modelArray);
- documentDeny = denyList.toArray(modelArray);
-
- // The format of this string changed on 11/8/2006 to be comformant
with the standard way
- // acls and metadata descriptions are being stuffed into the
version string across connectors.
-
- // The format of this string changed again on 7/3/2009 to permit
the ingestion uri/iri to be included.
- // This was to support filename/uri mapping functionality.
-
- StringBuilder sb = new StringBuilder();
-
- addSecuritySet(sb,shareSecurityOn,shareAllow,shareDeny);
- addSecuritySet(sb,parentSecurityOn,parentAllow,parentDeny);
- addSecuritySet(sb,documentSecurityOn,documentAllow,documentDeny);
+ if (fileIsDirectory)
+ {
+ // Hmm, this is not correct; version string should be empty for
windows directories, since
+ // they are not hierarchical in modified date propagation.
+ // It's a directory. The version ID will be the
+ // last modified date.
+ //long lastModified = fileLastModified(file);
+ //versionString = new Long(lastModified).toString();
+ versionString = "";
- // Include the path attribute name and value in the parseable area.
- if (pathAttributeName != null)
+ }
+ else
{
- sb.append('+');
- pack(sb,pathAttributeName,'+');
- // Calculate path string; we'll include that wholesale in the
version
- pathAttributeValue = documentIdentifier;
- // 3/13/2008
- // In looking at what comes into the path metadata attribute by
default, and cogitating a bit, I've concluded that
- // the smb:// and the server/domain name at the start of the
path are just plain old noise, and should be stripped.
- // This changes a behavior that has been around for a while, so
there is a risk, but a quick back-and-forth with the
- // SE's leads me to believe that this is safe.
+ fileLength = fileLength(file);
+ if (checkIncludeFile(fileLength,newPath,spec,activities))
+ {
+ // It's a file of acceptable length.
+ // The ability to get ACLs, list files, and an inputstream
under DFS all work now.
+ // The SmbFile for parentFolder acls.
+ SmbFile parentFolder = new SmbFile(file.getParent(),pa);
+
+ // Compute the security information
+ String[] modelArray = new String[0];
+
+ List<String> allowList = new ArrayList<String>();
+ List<String> denyList = new ArrayList<String>();
+ shareSecurityOn = getFileShareSecuritySet(allowList, denyList,
file, shareAcls);
+ shareAllow = allowList.toArray(modelArray);
+ shareDeny = denyList.toArray(modelArray);
+
+ allowList.clear();
+ denyList.clear();
+ parentSecurityOn = getFileSecuritySet(allowList, denyList,
parentFolder, parentFolderAcls);
+ parentAllow = allowList.toArray(modelArray);
+ parentDeny = denyList.toArray(modelArray);
+
+ allowList.clear();
+ denyList.clear();
+ documentSecurityOn = getFileSecuritySet(allowList, denyList,
file, acls);
+ documentAllow = allowList.toArray(modelArray);
+ documentDeny = denyList.toArray(modelArray);
+
+ // This is stuff we need for computing the version string AND
for indexing
+ lastModified = fileLastModified(file);
+
+ // The format of this string changed on 11/8/2006 to be
comformant with the standard way
+ // acls and metadata descriptions are being stuffed into the
version string across connectors.
+
+ // The format of this string changed again on 7/3/2009 to
permit the ingestion uri/iri to be included.
+ // This was to support filename/uri mapping functionality.
+
+ StringBuilder sb = new StringBuilder();
+
+ addSecuritySet(sb,shareSecurityOn,shareAllow,shareDeny);
+ addSecuritySet(sb,parentSecurityOn,parentAllow,parentDeny);
+
addSecuritySet(sb,documentSecurityOn,documentAllow,documentDeny);
+
+ // Include the path attribute name and value in the parseable
area.
+ if (pathAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,pathAttributeName,'+');
+ // Calculate path string; we'll include that wholesale in
the version
+ pathAttributeValue = documentIdentifier;
+ // 3/13/2008
+ // In looking at what comes into the path metadata attribute
by default, and cogitating a bit, I've concluded that
+ // the smb:// and the server/domain name at the start of the
path are just plain old noise, and should be stripped.
+ // This changes a behavior that has been around for a while,
so there is a risk, but a quick back-and-forth with the
+ // SE's leads me to believe that this is safe.
- if (pathAttributeValue.startsWith("smb://"))
+ if (pathAttributeValue.startsWith("smb://"))
+ {
+ int index =
pathAttributeValue.indexOf("/","smb://".length());
+ if (index == -1)
+ index = pathAttributeValue.length();
+ pathAttributeValue = pathAttributeValue.substring(index);
+ }
+ // Now, translate
+ pathAttributeValue = matchMap.translate(pathAttributeValue);
+ pack(sb,pathAttributeValue,'+');
+ }
+ else
+ sb.append('-');
+
+ // Calculate the ingestion IRI/URI, and include that in the
parseable area.
+ ingestionURI = convertToURI(documentIdentifier,fileMap,uriMap);
+ pack(sb,ingestionURI,'+');
+
+ // The stuff from here on down is non-parseable.
+ sb.append(new Long(lastModified).toString()).append(":")
+ .append(new Long(fileLength).toString());
+ // Also include the specification-based answer for the
question of whether fingerprinting is
+ // going to be done. Although we may not consider this to
truly be "version" information, the
+ // specification does affect whether anything is ingested or
not, so it really is. The alternative
+ // is to fingerprint right here, in the version part of the
world, but that's got a performance
+ // downside, because it means that we'd have to suck over
pretty much everything just to determine
+ // what we wanted to ingest.
+ boolean ifIndexable = wouldFileBeIncluded(newPath,spec,true);
+ boolean ifNotIndexable =
wouldFileBeIncluded(newPath,spec,false);
+ if (ifIndexable == ifNotIndexable)
+ sb.append("I");
+ else
+ sb.append(ifIndexable?"Y":"N");
+ versionString = sb.toString();
+ }
+ else
{
- int index = pathAttributeValue.indexOf("/","smb://".length());
- if (index == -1)
- index = pathAttributeValue.length();
- pathAttributeValue = pathAttributeValue.substring(index);
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
- // Now, translate
- pathAttributeValue = matchMap.translate(pathAttributeValue);
- pack(sb,pathAttributeValue,'+');
}
- else
- sb.append('-');
-
- // Calculate the ingestion IRI/URI, and include that in the
parseable area.
- ingestionURI = convertToURI(documentIdentifier,fileMap,uriMap);
- pack(sb,ingestionURI,'+');
-
- // The stuff from here on down is non-parseable.
- // Get the file's modified date.
- long lastModified = fileLastModified(file);
- sb.append(new Long(lastModified).toString()).append(":")
- .append(new Long(fileLength(file)).toString());
- // Also include the specification-based answer for the question of
whether fingerprinting is
- // going to be done. Although we may not consider this to truly
be "version" information, the
- // specification does affect whether anything is ingested or not,
so it really is. The alternative
- // is to fingerprint right here, in the version part of the world,
but that's got a performance
- // downside, because it means that we'd have to suck over pretty
much everything just to determine
- // what we wanted to ingest.
- boolean ifIndexable = wouldFileBeIncluded(newPath,spec,true);
- boolean ifNotIndexable = wouldFileBeIncluded(newPath,spec,false);
- if (ifIndexable == ifNotIndexable)
- sb.append("I");
- else
- sb.append(ifIndexable?"Y":"N");
- versionString = sb.toString();
+ }
+ else
+ {
+ activities.deleteDocument(documentIdentifier);
+ continue;
}
}
else
@@ -717,9 +740,9 @@ public class SharedDriveConnector extend
try
{
- if (fileExists(file))
+ if (fileExists)
{
- if (fileIsDirectory(file))
+ if (fileIsDirectory)
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("JCIFS: '"+documentIdentifier+"' is
a directory");
@@ -756,8 +779,9 @@ public class SharedDriveConnector extend
{
String uri = ingestionURI;
String fileNameString = file.getName();
- Date lastModifiedDate = new Date(file.lastModified());
+ Date lastModifiedDate = new Date(lastModified);
Date creationDate = new Date(file.createTime());
+ Long originalLength = new Long(fileLength);
String contentType = mapExtensionToMimeType(fileNameString);
if (!activities.checkURLIndexable(uri))
@@ -796,6 +820,8 @@ public class SharedDriveConnector extend
String shareName = file.getShare();
rd.setFileName(fileNameString);
+ rd.setOriginalSize(originalLength);
+
if (contentType != null)
rd.setMimeType(contentType);
rd.addField("lastModified", lastModifiedDate.toString());
@@ -857,7 +883,7 @@ public class SharedDriveConnector extend
if (checkIngest(tempFile, newPath, spec, activities))
{
- long fileLength = tempFile.length();
+ // Not needed; fetched earlier: long fileLength =
tempFile.length();
if (!activities.checkLengthIndexable(fileLength))
{
Logging.connectors.debug("JCIFS: Skipping file
because output connector cannot accept length ("+fileLength+")");
@@ -877,7 +903,7 @@ public class SharedDriveConnector extend
activities.ingestDocumentWithException(documentIdentifier, versionString, uri,
rd);
errorCode = "OK";
- fileLengthLong = new Long(tempFile.length());
+ fileLengthLong = new Long(fileLength);
}
finally
{
@@ -906,7 +932,7 @@ public class SharedDriveConnector extend
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("JCIFS: Local file data not
needed for '"+documentIdentifier+"'");
- long fileLength = fileLength(file);
+ // Not needed; fetched earlier: long fileLength =
fileLength(file);
if (!activities.checkLengthIndexable(fileLength))
{
Logging.connectors.debug("JCIFS: Skipping file because
output connector cannot accept length ("+fileLength+")");
@@ -927,7 +953,7 @@ public class SharedDriveConnector extend
activities.ingestDocumentWithException(documentIdentifier, versionString, uri,
rd);
errorCode = "OK";
- fileLengthLong = new Long(fileLength(file));
+ fileLengthLong = new Long(fileLength);
}
finally
{
@@ -1355,14 +1381,47 @@ public class SharedDriveConnector extend
// Protected methods
+ /** Check if a file's stats are OK for inclusion.
+ */
+ protected static boolean checkIncludeFile(long fileLength, String fileName,
Specification documentSpecification, IFingerprintActivity activities)
+ throws ManifoldCFException, ServiceInterruption
+ {
+ // If it's a file, make sure the maximum length is not exceeded
+ if (!activities.checkLengthIndexable(fileLength) ||
+ !activities.checkMimeTypeIndexable(mapExtensionToMimeType(fileName)))
+ return false;
+ long maxFileLength = Long.MAX_VALUE;
+ for (int i = 0; i < documentSpecification.getChildCount(); i++)
+ {
+ SpecificationNode sn = documentSpecification.getChild(i++);
+ if (sn.getType().equals(NODE_MAXLENGTH))
+ {
+ try
+ {
+ String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
+ if (value != null && value.length() > 0)
+ maxFileLength = new Long(value).longValue();
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ManifoldCFException("Bad number: "+e.getMessage(),e);
+ }
+ }
+ }
+ if (fileLength > maxFileLength)
+ return false;
+ return true;
+ }
+
+
/** Check if a file or directory should be included, given a document
specification.
- *@param file is the file object.
+ *@param isDirectory is true if the file is a directory.
*@param fileName is the canonical file name.
*@param documentSpecification is the specification.
*@return true if it should be included.
*/
- protected boolean checkInclude(SmbFile file, String fileName, Specification
documentSpecification, IFingerprintActivity activities)
- throws ManifoldCFException, ServiceInterruption
+ protected boolean checkInclude(boolean isDirectory, String fileName,
Specification documentSpecification)
+ throws ManifoldCFException
{
if (Logging.connectors.isDebugEnabled())
Logging.connectors.debug("JCIFS: In checkInclude for '"+fileName+"'");
@@ -1375,7 +1434,6 @@ public class SharedDriveConnector extend
{
String pathPart;
String filePart;
- boolean isDirectory = fileIsDirectory(file);
if (isDirectory)
{
@@ -1398,36 +1456,7 @@ public class SharedDriveConnector extend
}
}
- // If it's a file, make sure the maximum length is not exceeded
int i;
- if (!isDirectory)
- {
- long fileLength = fileLength(file);
- if (!activities.checkLengthIndexable(fileLength) ||
- !activities.checkMimeTypeIndexable(mapExtensionToMimeType(fileName)))
- return false;
- long maxFileLength = Long.MAX_VALUE;
- i = 0;
- while (i < documentSpecification.getChildCount())
- {
- SpecificationNode sn = documentSpecification.getChild(i++);
- if (sn.getType().equals(NODE_MAXLENGTH))
- {
- try
- {
- String value = sn.getAttributeValue(ATTRIBUTE_VALUE);
- if (value != null && value.length() > 0)
- maxFileLength = new Long(value).longValue();
- }
- catch (NumberFormatException e)
- {
- throw new ManifoldCFException("Bad number: "+e.getMessage(),e);
- }
- }
- }
- if (fileLength > maxFileLength)
- return false;
- }
// Scan until we match a startpoint
i = 0;
@@ -1539,25 +1568,11 @@ public class SharedDriveConnector extend
}
return false;
}
- catch (jcifs.smb.SmbAuthException e)
- {
- Logging.connectors.warn("JCIFS: Authorization exception checking
inclusion for "+fileName+" - skipping");
- return false;
- }
- catch (SmbException se)
- {
- processSMBException(se, fileName, "checking inclusion", "canonical path
mapping");
- return false;
- }
- catch (java.net.SocketTimeoutException e)
+ catch (MalformedURLException e)
{
throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
}
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (IOException e)
+ catch (UnknownHostException e)
{
throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
}
@@ -1687,15 +1702,11 @@ public class SharedDriveConnector extend
}
return false;
}
- catch (java.net.SocketTimeoutException e)
+ catch (MalformedURLException e)
{
throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
}
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (IOException e)
+ catch (UnknownHostException e)
{
throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
}
@@ -1835,25 +1846,11 @@ public class SharedDriveConnector extend
}
return false;
}
- catch (jcifs.smb.SmbAuthException e)
- {
- Logging.connectors.warn("JCIFS: Authorization exception checking
ingestion for "+fileName+" - skipping");
- return false;
- }
- catch (SmbException se)
- {
- processSMBException(se, fileName, "checking ingestion", "reading
document");
- return false;
- }
- catch (java.net.SocketTimeoutException e)
+ catch (MalformedURLException e)
{
throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
}
- catch (InterruptedIOException e)
- {
- throw new ManifoldCFException("Interrupted:
"+e.getMessage(),e,ManifoldCFException.INTERRUPTED);
- }
- catch (IOException e)
+ catch (UnknownHostException e)
{
throw new ManifoldCFException("Couldn't map to canonical path:
"+e.getMessage(),e);
}
@@ -2082,7 +2079,7 @@ public class SharedDriveConnector extend
/** Map a "path" specification to a full identifier.
*/
protected String mapToIdentifier(String path)
- throws IOException
+ throws MalformedURLException, UnknownHostException
{
String smburi = smbconnectionPath;
String uri = smburi + path + "/";
@@ -4820,11 +4817,30 @@ public class SharedDriveConnector extend
// documents that we will immediately turn around and remove.
However, if this
// check was not here, everything should still function, provided
the getDocumentVersions()
// method does the right thing.
- if (checkInclude(f, newPath, spec, activities))
+ boolean fileIsDirectory = fileIsDirectory(f);
+ if (checkInclude(fileIsDirectory, newPath, spec))
{
- if (Logging.connectors.isDebugEnabled())
- Logging.connectors.debug("JCIFS: Recorded path is '" + newPath +
"' and is included.");
- activities.addDocumentReference(newPath);
+ if (fileIsDirectory)
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("JCIFS: Recorded path is '" + newPath
+ "' and is included.");
+ activities.addDocumentReference(newPath);
+ }
+ else
+ {
+ long fileLength = fileLength(f);
+ if (checkIncludeFile(fileLength, newPath, spec, activities))
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("JCIFS: Recorded path is '" +
newPath + "' and is included.");
+ activities.addDocumentReference(newPath);
+ }
+ else
+ {
+ if (Logging.connectors.isDebugEnabled())
+ Logging.connectors.debug("JCIFS: Recorded path '"+newPath+"'
is excluded!");
+ }
+ }
}
else
{
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/HttpPoster.java
Fri May 29 10:04:57 2015
@@ -99,6 +99,7 @@ public class HttpPoster
private final String allowAttributeName;
private final String denyAttributeName;
private final String idAttributeName;
+ private final String originalSizeAttributeName;
private final String modifiedDateAttributeName;
private final String createdDateAttributeName;
private final String indexedDateAttributeName;
@@ -133,7 +134,7 @@ public class HttpPoster
int zkClientTimeout, int zkConnectTimeout,
String updatePath, String removePath, String statusPath,
String allowAttributeName, String denyAttributeName, String
idAttributeName,
- String modifiedDateAttributeName, String createdDateAttributeName, String
indexedDateAttributeName,
+ String originalSizeAttributeName, String modifiedDateAttributeName, String
createdDateAttributeName, String indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName, String
contentAttributeName,
Long maxDocumentLength,
String commitWithin, boolean useExtractUpdateHandler,
@@ -150,6 +151,7 @@ public class HttpPoster
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
+ this.originalSizeAttributeName = originalSizeAttributeName;
this.modifiedDateAttributeName = modifiedDateAttributeName;
this.createdDateAttributeName = createdDateAttributeName;
this.indexedDateAttributeName = indexedDateAttributeName;
@@ -182,7 +184,7 @@ public class HttpPoster
String updatePath, String removePath, String statusPath,
String realm, String userID, String password,
String allowAttributeName, String denyAttributeName, String
idAttributeName,
- String modifiedDateAttributeName, String createdDateAttributeName, String
indexedDateAttributeName,
+ String originalSizeAttributeName, String modifiedDateAttributeName, String
createdDateAttributeName, String indexedDateAttributeName,
String fileNameAttributeName, String mimeTypeAttributeName, String
contentAttributeName,
IKeystoreManager keystoreManager, Long maxDocumentLength,
String commitWithin, boolean useExtractUpdateHandler,
@@ -199,6 +201,7 @@ public class HttpPoster
this.allowAttributeName = allowAttributeName;
this.denyAttributeName = denyAttributeName;
this.idAttributeName = idAttributeName;
+ this.originalSizeAttributeName = originalSizeAttributeName;
this.modifiedDateAttributeName = modifiedDateAttributeName;
this.createdDateAttributeName = createdDateAttributeName;
this.indexedDateAttributeName = indexedDateAttributeName;
@@ -1020,6 +1023,14 @@ public class HttpPoster
}
// Write the rest of the attributes
+ if ( originalSizeAttributeName != null )
+ {
+ Long size = document.getOriginalSize();
+ if ( size != null )
+ {
+ outputDoc.addField( originalSizeAttributeName, size.toString() );
+ }
+ }
if ( modifiedDateAttributeName != null )
{
Date date = document.getModifiedDate();
@@ -1092,6 +1103,13 @@ public class HttpPoster
// Write the id field
writeField(out,LITERAL+idAttributeName,documentURI);
// Write the rest of the attributes
+ if (originalSizeAttributeName != null)
+ {
+ Long size = document.getOriginalSize();
+ if (size != null)
+ // Write value
+ writeField(out,LITERAL+modifiedDateAttributeName,size.toString());
+ }
if (modifiedDateAttributeName != null)
{
Date date = document.getModifiedDate();
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConfig.java
Fri May 29 10:04:57 2015
@@ -83,6 +83,8 @@ public class SolrConfig
public static final String PARAM_STATUSPATH = "Server status handler";
/** Id field */
public static final String PARAM_IDFIELD = "Solr id field name";
+ /** Optional original size field */
+ public static final String PARAM_ORIGINALSIZEFIELD = "Solr original size
field name";
/** Optional modified date field */
public static final String PARAM_MODIFIEDDATEFIELD = "Solr modified date
field name";
/** Optional created date field */
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/java/org/apache/manifoldcf/agents/output/solr/SolrConnector.java
Fri May 29 10:04:57 2015
@@ -85,6 +85,7 @@ public class SolrConnector extends org.a
// Attributes going into Solr
protected String idAttributeName = null;
+ protected String originalSizeAttributeName = null;
protected String modifiedDateAttributeName = null;
protected String createdDateAttributeName = null;
protected String indexedDateAttributeName = null;
@@ -183,6 +184,7 @@ public class SolrConnector extends org.a
excludedMimeTypesString = null;
excludedMimeTypes = null;
idAttributeName = null;
+ originalSizeAttributeName = null;
modifiedDateAttributeName = null;
createdDateAttributeName = null;
indexedDateAttributeName = null;
@@ -217,6 +219,10 @@ public class SolrConnector extends org.a
if (idAttributeName == null || idAttributeName.length() == 0)
idAttributeName = "id";
+ originalSizeAttributeName =
params.getParameter(SolrConfig.PARAM_ORIGINALSIZEFIELD);
+ if (originalSizeAttributeName == null ||
originalSizeAttributeName.length() == 0)
+ originalSizeAttributeName = null;
+
modifiedDateAttributeName =
params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
if (modifiedDateAttributeName == null ||
modifiedDateAttributeName.length() == 0)
modifiedDateAttributeName = null;
@@ -364,7 +370,7 @@ public class SolrConnector extends org.a
connectTimeout,socketTimeout,
updatePath,removePath,statusPath,realm,userID,password,
allowAttributeName,denyAttributeName,idAttributeName,
-
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
+
originalSizeAttributeName,modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler,
useUrlEncoding);
@@ -421,7 +427,7 @@ public class SolrConnector extends org.a
zkClientTimeout,zkConnectTimeout,
updatePath,removePath,statusPath,
allowAttributeName,denyAttributeName,idAttributeName,
-
modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
+
originalSizeAttributeName,modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,
fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,
maxDocumentLength,commitWithin,useExtractUpdateHandler,
useUrlEncoding);
@@ -1020,6 +1026,10 @@ public class SolrConnector extends org.a
String idField = parameters.getParameter(SolrConfig.PARAM_IDFIELD);
if (idField == null)
idField = "id";
+
+ String originalSizeField =
parameters.getParameter(SolrConfig.PARAM_ORIGINALSIZEFIELD);
+ if (originalSizeField == null)
+ originalSizeField = "";
String modifiedDateField =
parameters.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);
if (modifiedDateField == null)
@@ -1478,6 +1488,12 @@ public class SolrConnector extends org.a
" </td>\n"+
" </tr>\n"+
" <tr>\n"+
+" <td class=\"description\"><nobr>" +
Messages.getBodyString(locale,"SolrConnector.OriginalSizeFieldName") +
"</nobr></td>\n"+
+" <td class=\"value\">\n"+
+" <input name=\"originalsizefield\" type=\"text\" size=\"32\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(originalSizeField)+"\"/>\n"+
+" </td>\n"+
+" </tr>\n"+
+" <tr>\n"+
" <td class=\"description\"><nobr>" +
Messages.getBodyString(locale,"SolrConnector.ModifiedDateFieldName") +
"</nobr></td>\n"+
" <td class=\"value\">\n"+
" <input name=\"modifieddatefield\" type=\"text\" size=\"32\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
@@ -1563,6 +1579,7 @@ public class SolrConnector extends org.a
{
out.print(
"<input type=\"hidden\" name=\"idfield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(idField)+"\"/>\n"+
+"<input type=\"hidden\" name=\"originalsizefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(originalSizeField)+"\"/>\n"+
"<input type=\"hidden\" name=\"modifieddatefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(modifiedDateField)+"\"/>\n"+
"<input type=\"hidden\" name=\"createddatefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(createdDateField)+"\"/>\n"+
"<input type=\"hidden\" name=\"indexeddatefield\"
value=\""+org.apache.manifoldcf.ui.util.Encoder.attributeEscape(indexedDateField)+"\"/>\n"+
@@ -1853,6 +1870,10 @@ public class SolrConnector extends org.a
if (idField != null)
parameters.setParameter(SolrConfig.PARAM_IDFIELD,idField);
+ String originalSizeField =
variableContext.getParameter("originalsizefield");
+ if (originalSizeField != null)
+
parameters.setParameter(SolrConfig.PARAM_ORIGINALSIZEFIELD,originalSizeField);
+
String modifiedDateField =
variableContext.getParameter("modifieddatefield");
if (modifiedDateField != null)
parameters.setParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD,modifiedDateField);
@@ -2761,6 +2782,14 @@ public class SolrConnector extends org.a
}
else
sb.append('-');
+
+ if (originalSizeAttributeName != null)
+ {
+ sb.append('+');
+ pack(sb,originalSizeAttributeName,'+');
+ }
+ else
+ sb.append('-');
if (modifiedDateAttributeName != null)
{
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_en_US.properties
Fri May 29 10:04:57 2015
@@ -53,6 +53,7 @@ SolrConnector.Add=Add
SolrConnector.AddZookeeperHost=Add ZooKeeper host
SolrConnector.Certificate=Certificate:
SolrConnector.IDFieldName=ID field name:
+SolrConnector.OriginalSizeFieldName=Original size field name:
SolrConnector.ModifiedDateFieldName=Modified date field name:
SolrConnector.CreatedDateFieldName=Created date field name:
SolrConnector.IndexedDateFieldName=Indexed date field name:
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_ja_JP.properties
Fri May 29 10:04:57 2015
@@ -53,6 +53,7 @@ SolrConnector.Add=追å
SolrConnector.AddZookeeperHost=ZooKeeperãã¹ãã追å
SolrConnector.Certificate=証æ証ï¼
SolrConnector.IDFieldName=IDãã£ã¼ã«ãåï¼
+SolrConnector.OriginalSizeFieldName=Original size field name:
SolrConnector.ModifiedDateFieldName=æ´æ°æ¥ä»ãã£ã¼ã«ãåï¼
SolrConnector.CreatedDateFieldName=ä½ææ¥ä»ãã£ã¼ã«ãåï¼
SolrConnector.IndexedDateFieldName=Indexed date field name:
Modified:
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
(original)
+++
manifoldcf/branches/dev_1x/connectors/solr/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/solr/common_zh_CN.properties
Fri May 29 10:04:57 2015
@@ -53,6 +53,7 @@ SolrConnector.Add=æ·»å
SolrConnector.AddZookeeperHost=æ·»å ZooKeeper主æº
SolrConnector.Certificate=è¯ä¹¦:
SolrConnector.IDFieldName=IDå段å:
+SolrConnector.OriginalSizeFieldName=Original size field name:
SolrConnector.ModifiedDateFieldName=æ´æ°æ¥æå段å:
SolrConnector.CreatedDateFieldName=çææ¥æå段å:
SolrConnector.IndexedDateFieldName=ç´¢å¼åçæ¥æå段å:
Propchange: manifoldcf/branches/dev_1x/framework/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri May 29 10:04:57 2015
@@ -113,4 +113,4 @@
/manifoldcf/branches/CONNECTORS-989/framework:1611600-1612101
/manifoldcf/branches/CONNECTORS-990/framework:1610284-1610707
/manifoldcf/trunk:1629122
-/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605,1650351,1650911,1651332,1651539,1651921,1655377,1655411,1657346,1658004,1658036,1660258,1660276,1669487,1670614,1672616,1672737,1676094,1681390,1681735,1682232,1682252
+/manifoldcf/trunk/framework:1620703,1620748,1620812,1620862,1621449,1621613,1621855,1622213,1622740,1622850,1622853-1622854,1623249,1623251,1623314,1623599,1623951,1623953-1623954,1623956,1623972,1624058,1624085,1624174,1624236,1624377,1624384,1624399,1624449,1624464,1624504,1624729-1624731,1624906,1624909-1624910,1624982,1625023,1625095,1625103,1625108,1625264,1625270,1625394,1625400,1625910,1626090,1626097,1626102,1626638-1626639,1626973,1627687,1627690,1627959,1628046,1628066,1628106,1628168,1628188,1628699,1628798,1628808,1628845,1628905,1629122,1629374-1629375,1629379,1629541,1629994,1630188,1630535,1630623,1630671,1630812,1630885,1631039,1631162,1631164,1631252,1632013,1632289,1632844,1633108,1633193,1633202,1633348,1633364,1634145,1634148,1634155,1634264,1634373,1634530,1635438,1635809,1636146,1636180,1636207,1636232,1636334,1636519,1636570,1636684,1636940,1637011,1639593,1639600,1640018,1640101,1640199,1640314,1640319,1640749,1640772,1640925,1640941,1641222,1641557,1641559,1
641724,1641911,1642163,1642255,1642318,1644197,1644399,1646317,1646397,1646403,1646640,1647574,1647585,1647608,1649605,1650351,1650911,1651332,1651539,1651921,1655377,1655411,1657346,1658004,1658036,1660258,1660276,1669487,1670614,1672616,1672737,1676094,1681390,1681735,1682232,1682252,1682410
Modified:
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
URL:
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java?rev=1682411&r1=1682410&r2=1682411&view=diff
==============================================================================
---
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
(original)
+++
manifoldcf/branches/dev_1x/framework/agents/src/main/java/org/apache/manifoldcf/agents/interfaces/RepositoryDocument.java
Fri May 29 10:04:57 2015
@@ -56,6 +56,7 @@ public class RepositoryDocument
protected Date createdDate = null;
protected Date modifiedDate = null;
protected Date indexingDate = null;
+ protected Long originalSize = null;
/** Constructor.
*/
@@ -79,6 +80,7 @@ public class RepositoryDocument
rval.createdDate = createdDate;
rval.modifiedDate = modifiedDate;
rval.indexingDate = indexingDate;
+ rval.originalSize = originalSize;
for (String key : fields.keySet())
{
rval.fields.put(key,fields.get(key));
@@ -112,6 +114,23 @@ public class RepositoryDocument
readerFields.clear();
}
+ /** Set the document's original (repository) size. Use null to indicate
that the size is
+ * unknown.
+ *@param size is the size.
+ */
+ public void setOriginalSize(Long size)
+ {
+ originalSize = size;
+ }
+
+ /** Get the document's original size.
+ *@return the original repository document size, or null if unknown.
+ */
+ public Long getOriginalSize()
+ {
+ return originalSize;
+ }
+
/** Set the document's created date. Use null to indicate that the date is
unknown.
*@param date is the date.
*/
