svn commit: r1634859 - in /manifoldcf/branches/dev_1x: ./ connectors/tika/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tika/ connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/ connec...

Tue, 28 Oct 2014 05:41:00 -0700 

Author: kwright
Date: Tue Oct 28 12:40:05 2014
New Revision: 1634859

URL: http://svn.apache.org/r1634859
Log:
Pull up fix for CONNECTORS-1088 from trunk.

Added:
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Boilerplate.html
      - copied unchanged from r1634857, 
manifoldcf/trunk/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Boilerplate.html
Modified:
    manifoldcf/branches/dev_1x/   (props changed)
    manifoldcf/branches/dev_1x/CHANGES.txt
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tika/TikaExtractor.java
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_en_US.properties
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_ja_JP.properties
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_zh_CN.properties
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Exceptions.html
    
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/viewSpecification.html

Propchange: manifoldcf/branches/dev_1x/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1634857

Modified: manifoldcf/branches/dev_1x/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/CHANGES.txt?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- manifoldcf/branches/dev_1x/CHANGES.txt (original)
+++ manifoldcf/branches/dev_1x/CHANGES.txt Tue Oct 28 12:40:05 2014
@@ -3,6 +3,9 @@ $Id$
 
 ======================= 1.8-dev =====================
 
+CONNECTORS-1088: Add boilerplate extraction to Tika extractor.
+(Arcadius Ahouansou, Karl Wright)
+
 CONNECTORS-1087: Fix failing alfresco-webscript unit test.
 (Karl Wright)
 

Modified: 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tika/TikaExtractor.java
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tika/TikaExtractor.java?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tika/TikaExtractor.java
 (original)
+++ 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/java/org/apache/manifoldcf/agents/transformation/tika/TikaExtractor.java
 Tue Oct 28 12:40:05 2014
@@ -47,6 +47,7 @@ public class TikaExtractor extends org.a
   private static final String EDIT_SPECIFICATION_JS = "editSpecification.js";
   private static final String EDIT_SPECIFICATION_FIELDMAPPING_HTML = 
"editSpecification_FieldMapping.html";
   private static final String EDIT_SPECIFICATION_EXCEPTIONS_HTML = 
"editSpecification_Exceptions.html";
+  private static final String EDIT_SPECIFICATION_BOILERPLATE_HTML = 
"editSpecification_Boilerplate.html";
   private static final String VIEW_SPECIFICATION_HTML = 
"viewSpecification.html";
 
   protected static final String ACTIVITY_EXTRACT = "extract";
@@ -373,10 +374,12 @@ public class TikaExtractor extends org.a
 
     tabsArray.add(Messages.getString(locale, 
"TikaExtractor.FieldMappingTabName"));
     tabsArray.add(Messages.getString(locale, 
"TikaExtractor.ExceptionsTabName"));
+    tabsArray.add(Messages.getString(locale, 
"TikaExtractor.BoilerplateTabName"));
 
     // Fill in the specification header map, using data from all tabs.
     fillInFieldMappingSpecificationMap(paramMap, os);
     fillInExceptionsSpecificationMap(paramMap, os);
+    fillInBoilerplateSpecificationMap(paramMap, os);
     
     
Messages.outputResourceWithVelocity(out,locale,EDIT_SPECIFICATION_JS,paramMap);
   }
@@ -407,9 +410,11 @@ public class TikaExtractor extends org.a
     // Fill in the field mapping tab data
     fillInFieldMappingSpecificationMap(paramMap, os);
     fillInExceptionsSpecificationMap(paramMap, os);
-
+    fillInBoilerplateSpecificationMap(paramMap, os);
+    
     
Messages.outputResourceWithVelocity(out,locale,EDIT_SPECIFICATION_FIELDMAPPING_HTML,paramMap);
     
Messages.outputResourceWithVelocity(out,locale,EDIT_SPECIFICATION_EXCEPTIONS_HTML,paramMap);
+    
Messages.outputResourceWithVelocity(out,locale,EDIT_SPECIFICATION_BOILERPLATE_HTML,paramMap);
   }
 
   /** Process a specification post.
@@ -514,6 +519,27 @@ public class TikaExtractor extends org.a
       os.addChild(os.getChildCount(), node);
     }
     
+    x = variableContext.getParameter(seqPrefix+"boilerplateclassname");
+    if (x != null)
+    {
+      int i = 0;
+      while (i < os.getChildCount())
+      {
+        SpecificationNode node = os.getChild(i);
+        if (node.getType().equals(TikaConfig.NODE_BOILERPLATEPROCESSOR))
+          os.removeChild(i);
+        else
+          i++;
+      }
+
+      if (x.length() > 0)
+      {
+        SpecificationNode node = new 
SpecificationNode(TikaConfig.NODE_BOILERPLATEPROCESSOR);
+        node.setAttribute(TikaConfig.ATTRIBUTE_VALUE, x);
+        os.addChild(os.getChildCount(), node);
+      }
+    }
+    
     return null;
   }
   
@@ -537,6 +563,7 @@ public class TikaExtractor extends org.a
     // Fill in the map with data from all tabs
     fillInFieldMappingSpecificationMap(paramMap, os);
     fillInExceptionsSpecificationMap(paramMap, os);
+    fillInBoilerplateSpecificationMap(paramMap, os);
 
     
Messages.outputResourceWithVelocity(out,locale,VIEW_SPECIFICATION_HTML,paramMap);
     
@@ -590,6 +617,20 @@ public class TikaExtractor extends org.a
     paramMap.put("IGNORETIKAEXCEPTIONS",ignoreTikaExceptions);
   }
 
+  protected static void fillInBoilerplateSpecificationMap(Map<String,Object> 
paramMap, Specification os)
+  {
+    String boilerplateClassName = "";
+    for (int i = 0; i < os.getChildCount(); i++)
+    {
+      SpecificationNode sn = os.getChild(i);
+      if (sn.getType().equals(TikaConfig.NODE_BOILERPLATEPROCESSOR))
+      {
+        boilerplateClassName = 
sn.getAttributeValue(TikaConfig.ATTRIBUTE_VALUE);
+      }
+    }
+    paramMap.put("BOILERPLATECLASSNAME",boilerplateClassName);
+  }
+
   protected static int handleTikaException(TikaException e)
     throws IOException, ManifoldCFException, ServiceInterruption
   {

Modified: 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_en_US.properties
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_en_US.properties?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_en_US.properties
 (original)
+++ 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_en_US.properties
 Tue Oct 28 12:40:05 2014
@@ -15,6 +15,15 @@
 
 TikaExtractor.FieldMappingTabName=Field mapping
 TikaExtractor.ExceptionsTabName=Exceptions
+TikaExtractor.BoilerplateTabName=Boilerplate
+TikaExtractor.BoilerplateExtractorColon=Boilerplate extractor:
+TikaExtractor.NoExtractionSelected=-- No extraction selected --
+TikaExtractor.ExtractArticles=Extract articles
+TikaExtractor.ExtractArticleSentences=Extract article sentences
+TikaExtractor.BasicExtraction=Basic general-purpose extraction
+TikaExtractor.ExtractEverything=Extract everything
+TikaExtractor.ExtractLargestTextComponent=Extract the largest text component 
of the document
+TikaExtractor.ExtractNumWords=Extract based on number of words per block
 TikaExtractor.FieldMappings=Field mappings:
 TikaExtractor.MetadataFieldName=Metadata field name
 TikaExtractor.FinalFieldName=Final field name

Modified: 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_ja_JP.properties
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_ja_JP.properties?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_ja_JP.properties
 (original)
+++ 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_ja_JP.properties
 Tue Oct 28 12:40:05 2014
@@ -13,16 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-TikaExtractor.FieldMappingTabName=Field mapping
-TikaExtractor.ExceptionsTabName=Exceptions
-TikaExtractor.FieldMappings=Field mappings:
-TikaExtractor.MetadataFieldName=Metadata field name
-TikaExtractor.FinalFieldName=Final field name
-TikaExtractor.NoFieldMappingSpecified=No field mapping specified
-TikaExtractor.KeepAllMetadata=Keep all metadata:
-TikaExtractor.Add=Add
-TikaExtractor.AddFieldMapping=Add field mapping
-TikaExtractor.Delete=Delete
-TikaExtractor.DeleteFieldMapping=Delete field mapping
-TikaExtractor.NoFieldNameSpecified=Please specify a field name
-TikaExtractor.IgnoreTikaExceptions=Ignore Tika exceptions:
\ No newline at end of file
+TikaExtractor.FieldMappingTabName=フィールドマッピング
+TikaExtractor.ExceptionsTabName=例外
+TikaExtractor.BoilerplateTabName=Boilerplate
+TikaExtractor.BoilerplateExtractorColon=Boilerplate extractor:
+TikaExtractor.NoExtractionSelected=-- No extraction selected --
+TikaExtractor.ExtractArticles=Extract articles
+TikaExtractor.ExtractArticleSentences=Extract article sentences
+TikaExtractor.BasicExtraction=Basic general-purpose extraction
+TikaExtractor.ExtractEverything=Extract everything
+TikaExtractor.ExtractLargestTextComponent=Extract the largest text component 
of the document
+TikaExtractor.ExtractNumWords=Extract based on number of words per block
+TikaExtractor.FieldMappings=フィールドマッピング:
+TikaExtractor.MetadataFieldName=メタデータフィールド名
+TikaExtractor.FinalFieldName=最後のフィールド名
+TikaExtractor.NoFieldMappingSpecified=フィールドマッピングを指定してくã
 ã•ã„
+TikaExtractor.KeepAllMetadata=全メタデータを保存:
+TikaExtractor.Add=追加
+TikaExtractor.AddFieldMapping=フィールドマッピングを追加
+TikaExtractor.Delete=削除
+TikaExtractor.DeleteFieldMapping=フィールドマッピングを削除
+TikaExtractor.NoFieldNameSpecified=フィールド名を指定してくだ
さい
+TikaExtractor.IgnoreTikaExceptions=Tika例外を無視してください:

Modified: 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_zh_CN.properties
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_zh_CN.properties?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_zh_CN.properties
 (original)
+++ 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/native2ascii/org/apache/manifoldcf/agents/transformation/tika/common_zh_CN.properties
 Tue Oct 28 12:40:05 2014
@@ -13,16 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-TikaExtractor.FieldMappingTabName=Field mapping
-TikaExtractor.ExceptionsTabName=Exceptions
-TikaExtractor.FieldMappings=Field mappings:
-TikaExtractor.MetadataFieldName=Metadata field name
-TikaExtractor.FinalFieldName=Final field name
-TikaExtractor.NoFieldMappingSpecified=No field mapping specified
-TikaExtractor.KeepAllMetadata=Keep all metadata:
-TikaExtractor.Add=Add
-TikaExtractor.AddFieldMapping=Add field mapping
-TikaExtractor.Delete=Delete
-TikaExtractor.DeleteFieldMapping=Delete field mapping
-TikaExtractor.NoFieldNameSpecified=Please specify a field name
-TikaExtractor.IgnoreTikaExceptions=Ignore Tika exceptions:
\ No newline at end of file
+TikaExtractor.FieldMappingTabName=字段映射
+TikaExtractor.ExceptionsTabName=异常
+TikaExtractor.BoilerplateTabName=Boilerplate
+TikaExtractor.BoilerplateExtractorColon=Boilerplate extractor:
+TikaExtractor.NoExtractionSelected=-- No extraction selected --
+TikaExtractor.ExtractArticles=Extract articles
+TikaExtractor.ExtractArticleSentences=Extract article sentences
+TikaExtractor.BasicExtraction=Basic general-purpose extraction
+TikaExtractor.ExtractEverything=Extract everything
+TikaExtractor.ExtractLargestTextComponent=Extract the largest text component 
of the document
+TikaExtractor.ExtractNumWords=Extract based on number of words per block
+TikaExtractor.FieldMappings=字段映射:
+TikaExtractor.MetadataFieldName=元数据字段名
+TikaExtractor.FinalFieldName=最终字段名
+TikaExtractor.NoFieldMappingSpecified=未指定字段映射
+TikaExtractor.KeepAllMetadata=保存所有元数据:
+TikaExtractor.Add=添加
+TikaExtractor.AddFieldMapping=添加字段映射
+TikaExtractor.Delete=删除
+TikaExtractor.DeleteFieldMapping=删除字段映射
+TikaExtractor.NoFieldNameSpecified=未指定字段名
+TikaExtractor.IgnoreTikaExceptions=忽略Tika异常:

Modified: 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Exceptions.html
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Exceptions.html?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Exceptions.html
 (original)
+++ 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/editSpecification_Exceptions.html
 Tue Oct 28 12:40:05 2014
@@ -35,6 +35,6 @@
 #else
 
 <input type="hidden" name="s${SEQNUM}_ignoretikaexceptions_present" 
value="true"/>
-<input type="hidden" name="s${SEQNUM}_ignoretikaexceptions" 
value="$Encoder.bodyEscape($IGNORETIKAEXCEPTIONS)"/>
+<input type="hidden" name="s${SEQNUM}_ignoretikaexceptions" 
value="$Encoder.attributeEscape($IGNORETIKAEXCEPTIONS)"/>
 
 #end
\ No newline at end of file

Modified: 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/viewSpecification.html
URL: 
http://svn.apache.org/viewvc/manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/viewSpecification.html?rev=1634859&r1=1634858&r2=1634859&view=diff
==============================================================================
--- 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/viewSpecification.html
 (original)
+++ 
manifoldcf/branches/dev_1x/connectors/tika/connector/src/main/resources/org/apache/manifoldcf/agents/transformation/tika/viewSpecification.html
 Tue Oct 28 12:40:05 2014
@@ -56,5 +56,16 @@
     <td 
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.IgnoreTikaExceptions'))</nobr></td>
     <td 
class="value"><nobr>$Encoder.bodyEscape($IGNORETIKAEXCEPTIONS)</nobr></td>
   </tr>
+  <tr><td class="separator" colspan="2"><hr/></td></tr>
+  <tr>
+    <td 
class="description"><nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.BoilerplateExtractorColon'))</nobr></td>
+    <td class="value">
+#if ($BOILERPLATECLASSNAME == '')
+      
<nobr>$Encoder.bodyEscape($ResourceBundle.getString('TikaExtractor.NoExtractionSelected'))</nobr>
+#else
+      <nobr>$Encoder.bodyEscape($BOILERPLATECLASSNAME)</nobr>
+#end
+    </td>
+  </tr>
 
 </table>

Reply via email to