Author: rwesten
Date: Sat Nov 3 10:34:53 2012
New Revision: 1405317
URL: http://svn.apache.org/viewvc?rev=1405317&view=rev
Log:
Implementation of STANBOL-794; Changed Solr schema.xml for DBpedia indexing to
use omitNorms='false' to be compatible with Solr 3.6.1 (STANBOL-790)
Modified:
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/FieldValueFilter.java
stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/dbpedia/conf/schema.xml
stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/entityTypes.properties
Modified:
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/FieldValueFilter.java
URL:
http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/FieldValueFilter.java?rev=1405317&r1=1405316&r2=1405317&view=diff
==============================================================================
---
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/FieldValueFilter.java
(original)
+++
stanbol/trunk/entityhub/indexing/core/src/main/java/org/apache/stanbol/entityhub/indexing/core/processor/FieldValueFilter.java
Sat Nov 3 10:34:53 2012
@@ -60,7 +60,7 @@ public class FieldValueFilter implements
@Override
public Representation process(Representation source) {
- if(!includeEmpty && values.isEmpty()){ //no filter set
+ if(includeEmpty && values.isEmpty()){ //no filter set
return source;
}
Iterator<Reference> refs = source.getReferences(field);
@@ -68,7 +68,8 @@ public class FieldValueFilter implements
return source;
}
while(refs.hasNext()){
- if(values.contains(refs.next().getReference())){
+ //NOTE: if !includeEmpty values may be NULL (any value accepted)
+ if(values == null || values.contains(refs.next().getReference())){
return source;
}
}
@@ -100,12 +101,11 @@ public class FieldValueFilter implements
log.info("configured Field: {}",field);
}
value = config.get(PARAM_VALUES);
- if(value == null || value.toString().isEmpty()){
- throw new IllegalArgumentException("Missing required Parameter
"+PARAM_VALUES+". Set to '*' to deactivate Filtering");
- } else if(value instanceof String){
+ if(value instanceof String){
String stringValue = value.toString().trim();
if(stringValue.equals("*")){ // * -> deactivate Filtering
this.values = Collections.emptySet();
+ this.includeEmpty = true;
} else {
Set<String> values = new HashSet<String>();
for(String fieldValue : stringValue.split(";")){
@@ -128,6 +128,7 @@ public class FieldValueFilter implements
if(typeArray.length == 0 || //if an empty array or
typeArray.length == 1 && typeArray[0].equals("*")){ //only
a * is parsed
this.values = Collections.emptySet(); // than deactivate
filtering
+ this.includeEmpty = true;
} else {
Set<String> values = new HashSet<String>();
for(String filterString : typeArray){
@@ -145,9 +146,8 @@ public class FieldValueFilter implements
this.values = values;
}
}
- } else {
- throw new IllegalArgumentException("Type of parameter
"+PARAM_VALUES+'='+value+
- "(type:"+value.getClass()+") is not supported MUST be String
or String[]!");
+ } else {// no values (accept all entities with any value)
+ values = Collections.emptySet();
}
}
Modified:
stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/dbpedia/conf/schema.xml
URL:
http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/dbpedia/conf/schema.xml?rev=1405317&r1=1405316&r2=1405317&view=diff
==============================================================================
---
stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/dbpedia/conf/schema.xml
(original)
+++
stanbol/trunk/entityhub/indexing/dbpedia/src/main/resources/indexing/config/dbpedia/conf/schema.xml
Sat Nov 3 10:34:53 2012
@@ -69,7 +69,7 @@
<!-- boolean type: "true" or "false" used to store values with the
datatype "xsd:boolean" -->
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"
omitNorms="true"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"
omitNorms="false"/>
<!--Binary data type. The data should be sent/retrieved in as Base64
encoded Strings.
Currently not used by the SolrYard implementation, but reserved for
future use. -->
<fieldtype name="binary" class="solr.BinaryField"/>
@@ -83,11 +83,11 @@
for all numeric fields of that types. See Solr documentation for
suitable values and examples.
-->
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="date" class="solr.TrieDateField" omitNorms="true"
precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="false"
precisionStep="0" positionIncrementGap="0"/>
<!--
Numeric field types that index each value at various levels of precision
@@ -99,11 +99,11 @@
indexed per value, slightly larger index size, and faster range queries.
A precisionStep of 0 disables indexing at different precision levels.
-->
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8"
omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true"
precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8"
omitNorms="false" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="false"
precisionStep="6" positionIncrementGap="0"/>
<!-- not used
<fieldType name="random" class="solr.RandomSortField" indexed="true" />
Modified:
stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/entityTypes.properties
URL:
http://svn.apache.org/viewvc/stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/entityTypes.properties?rev=1405317&r1=1405316&r2=1405317&view=diff
==============================================================================
---
stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/entityTypes.properties
(original)
+++
stanbol/trunk/entityhub/indexing/genericrdf/src/main/resources/indexing/config/entityTypes.properties
Sat Nov 3 10:34:53 2012
@@ -38,11 +38,25 @@
#field=rdf:type
#The key 'values' is used to specify the filter
-# - This is a required configuration.
+# - If NOT present, than Entities with NO values for the field are filtered.
All
+# others are accepted
# - The value '*' deactivates filtering
# - Multiple types are supported. Configurations are separated by ';'
# - It is possible to use full URIs are prefix:local name for all prefixes
registered
# in 'org.apache.stanbol.entityhub.servicesapi.defaults.NamespaceEnum'
+# - null can be used to explicitly include Entities with no value
+
+#Examples
#This deactivate filtering
-values=*
\ No newline at end of file
+values=*
+
+#This activated filtering for Persons, Places and Organisation and also
includes
+#all entities with no type
+#values=null;dbp-ont:Person;dbp-ont:Place;dbp-ont:Organisation;
+
+#The following two configurations would only index entities with no values for
the
+#configured field
+#values=null
+#values=
+
