Updated Branches: refs/heads/develop 5f99c9e31 -> c2bea53f1
- custom externalizer for infinispan ldcache backend Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/c2bea53f Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/c2bea53f Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/c2bea53f Branch: refs/heads/develop Commit: c2bea53f1b7be2b4692e21219917d7136a6b3e09 Parents: 5f99c9e Author: Sebastian Schaffert <[email protected]> Authored: Mon Feb 3 18:14:17 2014 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Mon Feb 3 18:14:17 2014 +0100 ---------------------------------------------------------------------- .../registry/CacheTripleRegistry.java | 2 +- .../ldcache/ldcache-backend-infinispan/pom.xml | 9 +- .../infinispan/LDCachingInfinispanBackend.java | 64 ++++-- .../infinispan/io/ModelExternalizer.java | 150 ++++++++++++++ .../infinispan/io/ValueExternalizer.java | 197 +++++++++++++++++++ .../ldcache/backend/infinispan/util/DataIO.java | 68 +++++++ .../src/main/resources/jgroups-ldcache.xml | 2 +- 7 files changed, 471 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/registry/CacheTripleRegistry.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/registry/CacheTripleRegistry.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/registry/CacheTripleRegistry.java index 8d2d118..8f81531 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/registry/CacheTripleRegistry.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/registry/CacheTripleRegistry.java @@ -96,7 +96,7 @@ public class CacheTripleRegistry implements KiWiTripleRegistry { public void releaseTransaction(long transactionId) { if(transactions.containsKey(transactionId)) { for(long key : transactions.remove(transactionId)) { - cache.removeAsync(key); + cache.remove(key); } } } http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/ldcache/ldcache-backend-infinispan/pom.xml ---------------------------------------------------------------------- diff --git a/libraries/ldcache/ldcache-backend-infinispan/pom.xml b/libraries/ldcache/ldcache-backend-infinispan/pom.xml index 1069f73..a4bd526 100644 --- a/libraries/ldcache/ldcache-backend-infinispan/pom.xml +++ b/libraries/ldcache/ldcache-backend-infinispan/pom.xml @@ -42,6 +42,10 @@ <groupId>org.apache.marmotta</groupId> <artifactId>ldcache-api</artifactId> </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> <!-- Logging --> <dependency> @@ -111,11 +115,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/LDCachingInfinispanBackend.java ---------------------------------------------------------------------- diff --git a/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/LDCachingInfinispanBackend.java b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/LDCachingInfinispanBackend.java index 3d848d5..8d8a88a 100644 --- a/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/LDCachingInfinispanBackend.java +++ b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/LDCachingInfinispanBackend.java @@ -17,7 +17,10 @@ package org.apache.marmotta.ldcache.backend.infinispan; +import org.apache.commons.io.IOUtils; import org.apache.marmotta.ldcache.api.LDCachingBackend; +import org.apache.marmotta.ldcache.backend.infinispan.io.ModelExternalizer; +import org.apache.marmotta.ldcache.backend.infinispan.io.ValueExternalizer; import org.apache.marmotta.ldcache.model.CacheEntry; import org.infinispan.Cache; import org.infinispan.configuration.cache.CacheMode; @@ -34,6 +37,7 @@ import org.openrdf.model.URI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.concurrent.TimeUnit; /** @@ -84,24 +88,56 @@ public class LDCachingInfinispanBackend implements LDCachingBackend { } + + public LDCachingInfinispanBackend(String clusterName, String machineName) { + this(clusterName, machineName, 62333); + } + /** - * Create a clustered instane of the infinispan cache backend using the provided cluster and machine name + * Create a clustered instance of the infinispan cache backend using the provided cluster and machine name * @param clusterName * @param machineName + * @param clusterPort port to use for multicast messages */ - public LDCachingInfinispanBackend(String clusterName, String machineName) { - globalConfiguration = new GlobalConfigurationBuilder() - .classLoader(LDCachingInfinispanBackend.class.getClassLoader()) - .transport() - .defaultTransport() - .clusterName(clusterName) - .machineId(machineName) - .addProperty("configurationFile", "jgroups-ldcache.xml") - .globalJmxStatistics() - .jmxDomain("org.apache.marmotta.ldcache") - .allowDuplicateDomains(true) - .build(); - + public LDCachingInfinispanBackend(String clusterName, String machineName, int clusterPort) { + try { + String jgroupsXml = IOUtils.toString(LDCachingInfinispanBackend.class.getResourceAsStream("/jgroups-ldcache.xml")); + + jgroupsXml = jgroupsXml.replaceAll("mcast_port=\"[0-9]+\"", String.format("mcast_port=\"%d\"", clusterPort)); + + globalConfiguration = new GlobalConfigurationBuilder() + .classLoader(LDCachingInfinispanBackend.class.getClassLoader()) + .transport() + .defaultTransport() + .clusterName(clusterName) + .machineId(machineName) + .addProperty("configurationXml", jgroupsXml) + .globalJmxStatistics() + .jmxDomain("org.apache.marmotta.ldcache") + .allowDuplicateDomains(true) + .serialization() + .addAdvancedExternalizer(new ModelExternalizer()) + .addAdvancedExternalizer(new ValueExternalizer()) + .build(); + } catch (IOException ex) { + log.warn("error loading JGroups configuration from archive: {}", ex.getMessage()); + log.warn("some configuration options will not be available"); + + globalConfiguration = new GlobalConfigurationBuilder() + .classLoader(LDCachingInfinispanBackend.class.getClassLoader()) + .transport() + .defaultTransport() + .clusterName(clusterName) + .machineId(machineName) + .addProperty("configurationFile", "jgroups-ldcache.xml") + .globalJmxStatistics() + .jmxDomain("org.apache.marmotta.ldcache") + .allowDuplicateDomains(true) + .serialization() + .addAdvancedExternalizer(new ModelExternalizer()) + .addAdvancedExternalizer(new ValueExternalizer()) + .build(); + } defaultConfiguration = new ConfigurationBuilder() http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ModelExternalizer.java ---------------------------------------------------------------------- diff --git a/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ModelExternalizer.java b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ModelExternalizer.java new file mode 100644 index 0000000..017e557 --- /dev/null +++ b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ModelExternalizer.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.marmotta.ldcache.backend.infinispan.io; + +import org.infinispan.commons.marshall.AdvancedExternalizer; +import org.infinispan.commons.util.Util; +import org.openrdf.model.Resource; +import org.openrdf.model.Statement; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.impl.ContextStatementImpl; +import org.openrdf.model.impl.StatementImpl; +import org.openrdf.model.impl.TreeModel; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Set; + +/** + * Add file description here! + * + * @author Sebastian Schaffert ([email protected]) + */ +public class ModelExternalizer implements AdvancedExternalizer<TreeModel> { + + + /** + * Returns a collection of Class instances representing the types that this + * AdvancedExternalizer can marshall. Clearly, empty sets are not allowed. + * The externalizer framework currently requires all individual types to be + * listed since it does not make assumptions based on super classes or + * interfaces. + * + * @return A set containing the Class instances that can be marshalled. + */ + @Override + public Set<Class<? extends TreeModel>> getTypeClasses() { + return Util.<Class<? extends TreeModel>>asSet(TreeModel.class); + } + + /** + * Returns an integer that identifies the externalizer type. This is used + * at read time to figure out which {@link org.infinispan.commons.marshall.AdvancedExternalizer} should read + * the contents of the incoming buffer. + * <p/> + * Using a positive integer allows for very efficient variable length + * encoding of numbers, and it's much more efficient than shipping + * {@link org.infinispan.commons.marshall.AdvancedExternalizer} implementation class information around. + * Negative values are not allowed. + * <p/> + * Implementers of this interface can use any positive integer as long as + * it does not clash with any other identifier in the system. You can find + * information on the pre-assigned identifier ranges in + * <a href="http://community.jboss.org/docs/DOC-16198";>here</a>. + * <p/> + * It's highly recommended that maintaining of these identifiers is done + * in a centralized way and you can do so by making annotations reference + * a set of statically defined identifiers in a separate class or + * interface. Such class/interface gives a global view of the identifiers + * in use and so can make it easier to assign new ids. + * <p/> + * Implementors can optionally avoid giving a meaningful implementation to + * this method (i.e. return null) and instead rely on XML or programmatic + * configuration to provide the AdvancedExternalizer id. If no id can be + * determined via the implementation or XML/programmatic configuration, an + * error will be reported. If an id has been defined both via the + * implementation and XML/programmatic configuration, the value defined via + * XML/programmatic configuration will be used ignoring the other. + * + * @return A positive identifier for the AdvancedExternalizer. + */ + @Override + public Integer getId() { + return 265; + } + + /** + * Write the object reference to the stream. + * + * @param output the object output to write to + * @param object the object reference to write + * @throws java.io.IOException if an I/O error occurs + */ + @Override + public void writeObject(ObjectOutput output, TreeModel object) throws IOException { + output.writeInt(object.size()); + for(Statement statement : object) { + output.writeObject(statement.getSubject()); + output.writeObject(statement.getPredicate()); + output.writeObject(statement.getContext()); + if(statement.getContext() != null) { + output.writeBoolean(true); + output.writeObject(statement.getContext()); + } else { + output.writeBoolean(false); + } + + } + } + + /** + * Read an instance from the stream. The instance will have been written by the + * {@link #writeObject(java.io.ObjectOutput, Object)} method. Implementations are free + * to create instances of the object read from the stream in any way that they + * feel like. This could be via constructor, factory or reflection. + * + * @param input the object input to read from + * @return the object instance + * @throws java.io.IOException if an I/O error occurs + * @throws ClassNotFoundException if a class could not be found + */ + @Override + public TreeModel readObject(ObjectInput input) throws IOException, ClassNotFoundException { + TreeModel model = new TreeModel(); + + int size = input.readInt(); + for(int i=0; i<size; i++) { + Resource subject = (Resource) input.readObject(); + URI predicate = (URI) input.readObject(); + Value object = (Value) input.readObject(); + + boolean hasContext = input.readBoolean(); + if(hasContext) { + Resource context = (Resource) input.readObject(); + + model.add(new ContextStatementImpl(subject,predicate,object,context)); + } else { + model.add(new StatementImpl(subject,predicate,object)); + } + } + + return null; + } +} http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ValueExternalizer.java ---------------------------------------------------------------------- diff --git a/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ValueExternalizer.java b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ValueExternalizer.java new file mode 100644 index 0000000..09aab92 --- /dev/null +++ b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/io/ValueExternalizer.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.marmotta.ldcache.backend.infinispan.io; + +import org.apache.marmotta.ldcache.backend.infinispan.util.DataIO; +import org.infinispan.commons.marshall.AdvancedExternalizer; +import org.openrdf.model.BNode; +import org.openrdf.model.Literal; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.impl.BNodeImpl; +import org.openrdf.model.impl.LiteralImpl; +import org.openrdf.model.impl.URIImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.HashSet; +import java.util.Set; + +/** + * Add file description here! + * + * @author Sebastian Schaffert ([email protected]) + */ +public class ValueExternalizer implements AdvancedExternalizer<Value> { + + private static Logger log = LoggerFactory.getLogger(ValueExternalizer.class); + + private static final int TYPE_URI = 1; + private static final int TYPE_BNODE = 2; + private static final int TYPE_LITERAL = 3; + + + public ValueExternalizer() { + } + + + /** + * Returns a collection of Class instances representing the types that this + * AdvancedExternalizer can marshall. Clearly, empty sets are not allowed. + * The externalizer framework currently requires all individual types to be + * listed since it does not make assumptions based on super classes or + * interfaces. + * + * @return A set containing the Class instances that can be marshalled. + */ + @Override + public Set<Class<? extends Value>> getTypeClasses() { + Set<Class<? extends Value>> classes = new HashSet<>(); + classes.add(BNode.class); + classes.add(URI.class); + classes.add(Literal.class); + classes.add(BNodeImpl.class); + classes.add(URIImpl.class); + classes.add(LiteralImpl.class); + classes.add(Value.class); + + return classes; + } + + /** + * Returns an integer that identifies the externalizer type. This is used + * at read time to figure out which {@link org.infinispan.commons.marshall.AdvancedExternalizer} should read + * the contents of the incoming buffer. + * <p/> + * Using a positive integer allows for very efficient variable length + * encoding of numbers, and it's much more efficient than shipping + * {@link org.infinispan.commons.marshall.AdvancedExternalizer} implementation class information around. + * Negative values are not allowed. + * <p/> + * Implementers of this interface can use any positive integer as long as + * it does not clash with any other identifier in the system. You can find + * information on the pre-assigned identifier ranges in + * <a href="http://community.jboss.org/docs/DOC-16198";>here</a>. + * <p/> + * It's highly recommended that maintaining of these identifiers is done + * in a centralized way and you can do so by making annotations reference + * a set of statically defined identifiers in a separate class or + * interface. Such class/interface gives a global view of the identifiers + * in use and so can make it easier to assign new ids. + * <p/> + * Implementors can optionally avoid giving a meaningful implementation to + * this method (i.e. return null) and instead rely on XML or programmatic + * configuration to provide the AdvancedExternalizer id. If no id can be + * determined via the implementation or XML/programmatic configuration, an + * error will be reported. If an id has been defined both via the + * implementation and XML/programmatic configuration, the value defined via + * XML/programmatic configuration will be used ignoring the other. + * + * @return A positive identifier for the AdvancedExternalizer. + */ + @Override + public Integer getId() { + return 263; + } + + /** + * Write the object reference to the stream. + * + * @param out the object output to write to + * @param value the object reference to write + * @throws java.io.IOException if an I/O error occurs + */ + @Override + public void writeObject(ObjectOutput out, Value value) throws IOException { + int type = getType(value.getClass()); + + out.writeInt(type); + DataIO.writeString(out, value.stringValue()); + + if(type == TYPE_LITERAL) { + Literal l = (Literal)value; + + DataIO.writeString(out,l.getLanguage()); + + if(l.getDatatype() != null) { + DataIO.writeString(out,l.getDatatype().stringValue()); + } else { + DataIO.writeString(out,null); + } + } + + } + + /** + * Read an instance from the stream. The instance will have been written by the + * {@link #writeObject(java.io.ObjectOutput, Object)} method. Implementations are free + * to create instances of the object read from the stream in any way that they + * feel like. This could be via constructor, factory or reflection. + * + * @param in the object input to read from + * @return the object instance + * @throws java.io.IOException if an I/O error occurs + * @throws ClassNotFoundException if a class could not be found + */ + @Override + public Value readObject(ObjectInput in) throws IOException, ClassNotFoundException { + int type = in.readInt(); + + String label = DataIO.readString(in); + switch (type) { + case TYPE_URI: + return new URIImpl(label); + case TYPE_BNODE: + return new BNodeImpl(label); + case TYPE_LITERAL: + String lang = DataIO.readString(in); + String dtype = DataIO.readString(in); + + if(lang != null) { + return new LiteralImpl(label,lang); + } else if(dtype != null) { + return new LiteralImpl(label, new URIImpl(dtype)); + } else { + return new LiteralImpl(label); + } + } + throw new ClassNotFoundException("could not find class with type "+type); + } + + private static <C1 extends Value, C2 extends Value> int compareTypes(Class<C1> clazz1, Class<C2> clazz2) { + int t1 = getType(clazz1), t2 = getType(clazz2); + + return t1 - t2; + } + + + private static <C extends Value> int getType(Class<C> clazz) { + int t = 0; + if(URI.class.isAssignableFrom(clazz)) { + t = TYPE_URI; + } else if(BNode.class.isAssignableFrom(clazz)) { + t = TYPE_BNODE; + } else { + t = TYPE_LITERAL; + } + return t; + } +} http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/util/DataIO.java ---------------------------------------------------------------------- diff --git a/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/util/DataIO.java b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/util/DataIO.java new file mode 100644 index 0000000..687b9a1 --- /dev/null +++ b/libraries/ldcache/ldcache-backend-infinispan/src/main/java/org/apache/marmotta/ldcache/backend/infinispan/util/DataIO.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.marmotta.ldcache.backend.infinispan.util; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Convenience methods for efficiently reading/writing strings from data streams. + * + * @author Sebastian Schaffert ([email protected]) + */ +public class DataIO { + + /** + * Write a string to a DataOutput. Will first write an integer indicating the string length, followed + * by the characters contained in the string. In case the string is "null", writes length -1. + * + * @param out + * @param s + * @throws IOException + */ + public static void writeString(DataOutput out, String s) throws IOException { + if(s != null) { + out.writeInt(s.length()); + out.writeChars(s); + } else { + out.writeInt(-1); + } + } + + /** + * Read a string from a DataInput in the format produced by writeString. If the length is -1, returns null. + * + * @param in + * @return + * @throws IOException + */ + public static String readString(DataInput in) throws IOException { + int len = in.readInt(); + + if(len >= 0) { + char[] result = new char[len]; + for(int i=0; i<len; i++) { + result[i] = in.readChar(); + } + return new String(result); + } else { + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/marmotta/blob/c2bea53f/libraries/ldcache/ldcache-backend-infinispan/src/main/resources/jgroups-ldcache.xml ---------------------------------------------------------------------- diff --git a/libraries/ldcache/ldcache-backend-infinispan/src/main/resources/jgroups-ldcache.xml b/libraries/ldcache/ldcache-backend-infinispan/src/main/resources/jgroups-ldcache.xml index 1b1b0bd..97cd41f 100644 --- a/libraries/ldcache/ldcache-backend-infinispan/src/main/resources/jgroups-ldcache.xml +++ b/libraries/ldcache/ldcache-backend-infinispan/src/main/resources/jgroups-ldcache.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/JGroups-3.4.xsd";> <UDP mcast_addr="${jgroups.udp.mcast_addr:228.6.7.8}" - mcast_port="${jgroups.udp.mcast_port:46656}" + mcast_port="46656" tos="8" ucast_recv_buf_size="20m" ucast_send_buf_size="640k"
