Modified: incubator/stanbol/trunk/rick/indexing/geonames/pom.xml URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/indexing/geonames/pom.xml?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/indexing/geonames/pom.xml (original) +++ incubator/stanbol/trunk/rick/indexing/geonames/pom.xml Sun Dec 5 18:00:18 2010 @@ -21,7 +21,7 @@ </parent> --> <groupId>eu.iksproject</groupId> - <artifactId>eu.iksproject.indexing.geonames</artifactId> + <artifactId>eu.iksproject.rick.indexing.geonames</artifactId> <packaging>jar</packaging> <version>0.1-SNAPSHOT</version> <name>IKS RICK Indexing for geonames.org</name>
Propchange: incubator/stanbol/trunk/rick/jersey/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,7 @@ +target + +.project + +.settings + +.classpath Modified: incubator/stanbol/trunk/rick/jersey/pom.xml URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/jersey/pom.xml?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/jersey/pom.xml (original) +++ incubator/stanbol/trunk/rick/jersey/pom.xml Sun Dec 5 18:00:18 2010 @@ -181,12 +181,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>eu.iksproject</groupId> - <artifactId>eu.iksproject.fise.jobmanager</artifactId> - <version>0.9-SNAPSHOT</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.7</version> Modified: incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpoint.java URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpoint.java?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpoint.java (original) +++ incubator/stanbol/trunk/rick/jersey/src/main/java/eu/iksproject/rick/jersey/JerseyEndpoint.java Sun Dec 5 18:00:18 2010 @@ -39,17 +39,17 @@ public class JerseyEndpoint { private final Logger log = LoggerFactory.getLogger(getClass()); - @Property(value = "/") - public static final String ALIAS_PROPERTY = "eu.iksproject.fise.jersey.alias"; + @Property(value = "/rick") + public static final String ALIAS_PROPERTY = "eu.iksproject.rick.jersey.alias"; - @Property(value = "/static") - public static final String STATIC_RESOURCES_URL_ROOT_PROPERTY = "eu.iksproject.fise.jersey.static.url"; + @Property(value = "/rick/static") + public static final String STATIC_RESOURCES_URL_ROOT_PROPERTY = "eu.iksproject.rick.jersey.static.url"; - @Property(value = "/META-INF/static") - public static final String STATIC_RESOURCES_CLASSPATH_PROPERTY = "eu.iksproject.fise.jersey.static.classpath"; + @Property(value = "/rick/META-INF/static") + public static final String STATIC_RESOURCES_CLASSPATH_PROPERTY = "eu.iksproject.rick.jersey.static.classpath"; //@Property(value = "/META-INF/templates") - //public static final String FREEMARKER_TEMPLATE_CLASSPATH_PROPERTY = "eu.iksproject.fise.jersey.templates.classpath"; + //public static final String FREEMARKER_TEMPLATE_CLASSPATH_PROPERTY = "eu.iksproject.rick.jersey.templates.classpath"; @Reference TcManager tcManager; Propchange: incubator/stanbol/trunk/rick/launchers/sling/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,7 @@ +.settings + +target + +.classpath + +.project Propchange: incubator/stanbol/trunk/rick/model/clerezza/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,7 @@ +.settings + +target + +.project + +.classpath Modified: incubator/stanbol/trunk/rick/pom.xml URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/pom.xml?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/pom.xml (original) +++ incubator/stanbol/trunk/rick/pom.xml Sun Dec 5 18:00:18 2010 @@ -59,7 +59,6 @@ <!-- Utils for createing local caches (indexing utils) --> <module>indexing/geonames</module> <module>indexing/genericRdf</module> - <!-- disabling missing stuff <module>jersey</module> --> Propchange: incubator/stanbol/trunk/rick/query/clerezza/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,7 @@ +.settings + +target + +.project + +.classpath Propchange: incubator/stanbol/trunk/rick/site/linkedData/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,7 @@ +target + +.settings + +.classpath + +.project Propchange: incubator/stanbol/trunk/rick/yard/clerezza/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,3 @@ +target + +.settings Propchange: incubator/stanbol/trunk/rick/yard/solr/ ------------------------------------------------------------------------------ --- svn:ignore (added) +++ svn:ignore Sun Dec 5 18:00:18 2010 @@ -0,0 +1,7 @@ +target + +.settings + +.classpath + +.project Modified: incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/SolrConst.java URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/SolrConst.java?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/SolrConst.java (original) +++ incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/defaults/SolrConst.java Sun Dec 5 18:00:18 2010 @@ -79,5 +79,9 @@ public class SolrConst { * Documents with the same ID (equal values for {...@link #DOCUMENT_ID_FIELD}) */ public static final String DOMAIN_FIELD = SPECIAL_FIELD_PREFIX+"domain"; + /** + * The field name used by Solr for the score of query results + */ + public static final String SCORE_FIELD = "score"; } Modified: incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrFieldMapper.java URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrFieldMapper.java?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrFieldMapper.java (original) +++ incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrFieldMapper.java Sun Dec 5 18:00:18 2010 @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import eu.iksproject.rick.core.utils.ModelUtils; import eu.iksproject.rick.servicesapi.defaults.NamespaceEnum; +import eu.iksproject.rick.servicesapi.model.rdf.RdfResourceEnum; import eu.iksproject.rick.yard.solr.defaults.IndexDataTypeEnum; import eu.iksproject.rick.yard.solr.defaults.SolrConst; import eu.iksproject.rick.yard.solr.model.FieldMapper; @@ -62,6 +63,13 @@ public class SolrFieldMapper implements */ private static final int LRU_MAPPINGS_CACHE_SIZE = 1024; /** + * The IndexField for the Solr score. This field is mapped to the field + * {...@link RdfResourceEnum#resultScore} and uses {...@link IndexDataTypeEnum#FLOAT} + */ + private static final IndexField scoreField = new IndexField( + Collections.singletonList(RdfResourceEnum.resultScore.getUri()), + IndexDataTypeEnum.FLOAT.getIndexType()); + /** * Internally used as LRU Cache with {...@link SolrFieldMapper#LRU_MAPPINGS_CACHE_SIZE} * elements. This subclass of {...@link LinkedHashMap} overrides the * {...@link LinkedHashMap#removeEldestEntry(Entry)} as suggested by the java @@ -121,7 +129,9 @@ public class SolrFieldMapper implements // meaning, that this index document field does not represent // an logical IndexField and should be ignored return null; - } + } else if (SolrConst.SCORE_FIELD.equals(fieldName)){ + return scoreField; + } //parse the prefix and suffix String[] tokens = fieldName.split(Character.toString(SolrConst.PATH_SEPERATOR)); int numTokens = tokens.length; @@ -509,7 +519,7 @@ public class SolrFieldMapper implements * {...@link #NAMESPACE_PREFIX_SEPERATOR_CHAR} this method returns the parsed * String.<p> * The local name may contain the {...@link #NAMESPACE_PREFIX_SEPERATOR_CHAR} - * Ã...@link #NAMESPACE_PREFIX_SEPERATOR_CHAR}'. The prefix MUST NOT contain + * {...@link #NAMESPACE_PREFIX_SEPERATOR_CHAR}'. The prefix MUST NOT contain * this char, because {...@link String#indexOf(int)} is used to split prefix * and local name. * @param shortFieldName the short name Modified: incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrQueryFactory.java URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrQueryFactory.java?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrQueryFactory.java (original) +++ incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrQueryFactory.java Sun Dec 5 18:00:18 2010 @@ -176,6 +176,8 @@ public class SolrQueryFactory { String.format("Unknown SELECT status %s! Adapt this implementation to the new value of the Enumeration", select)); } + //add the select for the score + query.addField("score"); } private IndexConstraint createIndexConstraint(Entry<String, Constraint> fieldConstraint){ IndexConstraint indexConstraint = new IndexConstraint(Arrays.asList(fieldConstraint.getKey())); Modified: incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYard.java URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYard.java?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYard.java (original) +++ incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYard.java Sun Dec 5 18:00:18 2010 @@ -8,7 +8,9 @@ import java.util.Dictionary; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.Map.Entry; import org.apache.felix.scr.annotations.Activate; import org.apache.felix.scr.annotations.Component; @@ -31,12 +33,15 @@ import org.osgi.service.component.Compon import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.sun.org.apache.bcel.internal.classfile.Code; + import eu.iksproject.rick.core.model.InMemoryValueFactory; import eu.iksproject.rick.core.query.DefaultQueryFactory; import eu.iksproject.rick.core.query.QueryResultListImpl; import eu.iksproject.rick.core.utils.AdaptingIterator; import eu.iksproject.rick.core.yard.AbstractYard; import eu.iksproject.rick.servicesapi.model.Representation; +import eu.iksproject.rick.servicesapi.model.rdf.RdfResourceEnum; import eu.iksproject.rick.servicesapi.query.Constraint; import eu.iksproject.rick.servicesapi.query.FieldQuery; import eu.iksproject.rick.servicesapi.query.QueryResultList; @@ -128,6 +133,20 @@ public class SolrYard extends AbstractYa */ public static final String MAX_BOOLEAN_CLAUSES = "eu.iksproject.rick.yard.solr.maxBooleanClauses"; /** + * This property allows to define a field that is used to parse the boost + * for the parsed representation. Typically this will be the pageRank of + * that entity within the referenced site (e.g. {...@link Math#log1p(double)} + * of the number of incoming links + */ + public static final String DOCUMENT_BOOST_FIELD = "eu.iksproject.rick.yard.solr.documentBoost"; + /** + * Key used to configure {...@link Entry Entry<String,Float>} for fields + * with the boost. If no Map is configured or a field is not present in the + * Map, than 1.0f is used as Boost. If a Document boost is present than the + * boost of a Field is documentBoost*fieldBoost. + */ + public static final String FIELD_BOOST_MAPPINGS = "eu.iksproject.rick.yard.solr.fieldBoosts"; + /** * The default value for the maxBooleanClauses of SolrQueries. Set to * {...@value #defaultMaxBooleanClauses} the default of Slor 1.4 */ @@ -169,7 +188,22 @@ public class SolrYard extends AbstractYa * */ private SolrQueryFactory solrQueryFactoy; - + /** + * Used to store the name of the field used to get the + * {...@link SolrInputDocument#setDocumentBoost(float)} for a Representation. + * This name is available via {...@link SolrYardConfig#getDocumentBoostFieldName()} + * however it is stored here to prevent lookups for field of every + * stored {...@link Representation}. + */ + private String documentBoostFieldName; + /** + * Map used to store boost values for fields. The default Boost for fields + * is 1.0f. This is used if this map is <code>null</code>, a field is not + * a key in this map, the value of a field in that map is <code>null</code> or + * lower equals zero. Also NOTE that the boost for fields is multiplied with + * the boost for the Document if present. + */ + private Map<String,Float> fieldBoostMap; /** * Default constructor as used by the OSGI environment.<p> DO NOT USE to * manually create instances! The SolrYard instances do need to be configured. @@ -230,6 +264,8 @@ public class SolrYard extends AbstractYa } solrQueryFactoy.setDefaultQueryResults(this.config.getDefaultQueryResultNumber()); solrQueryFactoy.setMaxQueryResults(this.config.getMaxQueryResultNumber()); + this.documentBoostFieldName = config.getDocumentBoostFieldName(); + this.fieldBoostMap = config.getFieldBoosts(); } @Deactivate protected void deactivate(ComponentContext context) { @@ -245,6 +281,8 @@ public class SolrYard extends AbstractYa this.fieldMapper = null; this.indexValueFactory = null; this.solrQueryFactoy = null; + this.documentBoostFieldName = null; + this.fieldBoostMap = null; super.deactivate(); //deactivate the super implementation } @@ -256,12 +294,14 @@ public class SolrYard extends AbstractYa private QueryResultList<Representation> find(final FieldQuery parsedQuery,SELECT select) throws YardException { log.debug(String.format("find %s",parsedQuery)); long start = System.currentTimeMillis(); - SolrQuery query = solrQueryFactoy.parseFieldQuery(parsedQuery,SELECT.QUERY); + SolrQuery query = solrQueryFactoy.parseFieldQuery(parsedQuery,select); long queryGeneration = System.currentTimeMillis(); final Set<String> selected; if(select == SELECT.QUERY){ //if query set the fields to add to the result Representations selected = parsedQuery.getSelectedFields(); + //add the score to query results! + selected.add(RdfResourceEnum.resultScore.getUri()); } else { //otherwise add all fields selected = null; @@ -509,6 +549,7 @@ public class SolrYard extends AbstractYa return representations; } /** + * boost if present! * @param representation * @return */ @@ -520,8 +561,14 @@ public class SolrYard extends AbstractYa inputDocument.addField(fieldMapper.getDocumentDomainField(), config.getId()); } // else we need to do nothing inputDocument.addField(fieldMapper.getDocumentIdField(), representation.getId()); + //first process the document boost + float documentBoost = documentBoostFieldName == null ? 1.0f : getDocumentBoost(representation); for(Iterator<String> fields = representation.getFieldNames();fields.hasNext();){ + //TODO: maybe add some functionality to prevent indexing of the + // field configured as documentBoostFieldName! String field = fields.next(); + Float fieldBoost = fieldBoostMap == null ? null : fieldBoostMap.get(field); + float boost = fieldBoost == null ? documentBoost : fieldBoost >= 0 ? fieldBoost * documentBoost: documentBoost; // log.debug(String.format(" > Process Representation Field %s",field)); for(Iterator<Object> values = representation.get(field);values.hasNext();){ //now we need to get the indexField for the value @@ -529,9 +576,9 @@ public class SolrYard extends AbstractYa IndexValue value; try { value = indexValueFactory.createIndexValue(next); - for(String fieldName :fieldMapper.getFieldNames(Arrays.asList(field), value)){ + for(String fieldName : fieldMapper.getFieldNames(Arrays.asList(field), value)){ // log.debug(String.format(" - add: %s=%s",fieldName,value)); - inputDocument.addField(fieldName, value.getValue()); + inputDocument.addField(fieldName, value.getValue(),boost); } }catch(Exception e){ log.warn(String.format("Unable to process value %s (type:%s) for field %s!",next,next.getClass(),field),e); @@ -540,6 +587,30 @@ public class SolrYard extends AbstractYa } return inputDocument; } + /** + * Extracts the document boost from a {...@link Representation}. + * @param representation the representation + * @return the Boost or <code>null</code> if not found or lower equals zero + */ + private float getDocumentBoost(Representation representation) { + if(documentBoostFieldName == null){ + return 1.0f; + } + Float documentBoost = null; + for(Iterator<Object> values =representation.get(documentBoostFieldName);values.hasNext() && documentBoost == null;){ + Object value = values.next(); + if(value instanceof Float){ + documentBoost = (Float) value; + } else { + try { + documentBoost = Float.parseFloat(value.toString()); + } catch (NumberFormatException e) { + log.warn(String.format("Unable to parse the Document Boost from field %s=%s[type=%s] -> The Document Boost MUST BE a Float value!",documentBoostFieldName,value,value.getClass())); + } + } + } + return documentBoost == null? 1.0f : documentBoost >= 0 ? documentBoost : 1.0f; + } @Override public Representation update(Representation representation) throws YardException { Modified: incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYardConfig.java URL: http://svn.apache.org/viewvc/incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYardConfig.java?rev=1042390&r1=1042389&r2=1042390&view=diff ============================================================================== --- incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYardConfig.java (original) +++ incubator/stanbol/trunk/rick/yard/solr/src/main/java/eu/iksproject/rick/yard/solr/impl/SolrYardConfig.java Sun Dec 5 18:00:18 2010 @@ -3,6 +3,7 @@ package eu.iksproject.rick.yard.solr.imp import java.net.MalformedURLException; import java.net.URL; import java.util.Dictionary; +import java.util.Map; import org.apache.solr.client.solrj.SolrServer; import org.osgi.service.cm.ConfigurationException; @@ -150,6 +151,35 @@ public final class SolrYardConfig extend config.put(SolrYard.MAX_BOOLEAN_CLAUSES, integer); } } + public void setDocumentBoostFieldName(String fieldName){ + if(fieldName == null || fieldName.isEmpty()){ + config.remove(SolrYard.DOCUMENT_BOOST_FIELD); + } else { + config.put(SolrYard.DOCUMENT_BOOST_FIELD, fieldName); + } + } + public String getDocumentBoostFieldName(){ + Object name = config.get(SolrYard.DOCUMENT_BOOST_FIELD); + return name == null?null:name.toString(); + } + public void setFieldBoosts(Map<String,Float> fieldBoosts){ + if(fieldBoosts != null){ + config.put(SolrYard.FIELD_BOOST_MAPPINGS, fieldBoosts); + } else { + config.remove(SolrYard.FIELD_BOOST_MAPPINGS); + } + } + public Map<String,Float> getFieldBoosts(){ + Object fieldBoosts = config.get(SolrYard.FIELD_BOOST_MAPPINGS); + if(fieldBoosts == null){ + return null; + } else if(fieldBoosts instanceof Map<?, ?>){ + return (Map<String,Float>)fieldBoosts; + } else { + //TODO: add support for parsing from String[] and Collection<String> + return null; + } + } /** * checks for the {...@link SolrYard#SOLR_SERVER_URI} */
