Author: bdelacretaz
Date: Mon Jul 26 12:21:43 2010
New Revision: 979260
URL: http://svn.apache.org/viewvc?rev=979260&view=rev
Log:
SLING-505 - use hierarchical paths to store streams, to avoid large number of
child nodes
Added:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
(with props)
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
(with props)
Modified:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
Modified:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
URL:
http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
---
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
(original)
+++
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/BackgroundTestServlet.java
Mon Jul 26 12:21:43 2010
@@ -53,7 +53,7 @@ public class BackgroundTestServlet exten
final PrintWriter w = response.getWriter();
final int cycles = getIntParam(request, "cycles", 10);
- final int interval = getIntParam(request, "interval", 1);
+ final int interval = getIntParam(request, "interval", 1000);
final int flushEvery = getIntParam(request, "flushEvery", 2);
w.println("Start at " + new Date());
@@ -65,7 +65,7 @@ public class BackgroundTestServlet exten
}
w.printf("Cycle %d of %d\n", i, cycles);
try {
- Thread.sleep(interval * 1000);
+ Thread.sleep(interval);
} catch (InterruptedException iex) {
throw new ServletException("InterruptedException", iex);
}
Modified:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
URL:
http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
---
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
(original)
+++
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeInputStream.java
Mon Jul 26 12:21:43 2010
@@ -39,31 +39,30 @@ public class NodeInputStream extends Inp
/** The Node under which we read our data */
private final Node node;
- /** Counter used to build the name of Property from
- * which we currently read */
- private int counter;
+ /** Computes path for stream storage */
+ private final NodeStreamPath streamPath;
/** Current stream that we are reading */
private InputStream currentStream;
public NodeInputStream(Node n) throws IOException {
node = n;
+ streamPath = new NodeStreamPath();
selectNextStream();
}
/** Select next property to read from and open its stream */
private void selectNextStream() throws IOException {
- counter++;
- // TODO use hierarchy to allow for arbitrary number of flush calls
- final String name = NodeOutputStream.STREAM_PROPERTY_NAME_PREFIX +
counter;
+ streamPath.selectNextPath();
+ final String propertyPath = streamPath.getNodePath() + "/" +
NodeStreamPath.PROPERTY_NAME;
try {
- if(node.hasProperty(name)) {
- final Property p = node.getProperty(name);
+ if(node.hasProperty(propertyPath)) {
+ final Property p = node.getProperty(propertyPath);
currentStream = p.getStream();
log.debug("Switched to the InputStream of Property {}",
p.getPath());
} else {
currentStream = null;
- log.debug("Property {} not found, end of stream",
node.getPath() + "/" + name);
+ log.debug("Property {} not found, end of stream",
node.getPath() + "/" + propertyPath);
}
} catch(RepositoryException re) {
throw new IOException("RepositoryException in
selectNextProperty()", re);
Modified:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
URL:
http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
---
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
(original)
+++
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeOutputStream.java
Mon Jul 26 12:21:43 2010
@@ -26,6 +26,7 @@ import java.io.OutputStream;
import javax.jcr.Node;
import javax.jcr.RepositoryException;
+import org.apache.sling.bgservlets.impl.DeepNodeCreator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,12 +51,14 @@ public class NodeOutputStream extends Ou
/** Prefix for Property names used to store our streams */
public static final String STREAM_PROPERTY_NAME_PREFIX = "_NODE_STREAM_";
+ /** Node type for our stream nodes */
+ public static final String STREAM_NODE_TYPE = "nt:unstructured";
+
/** The Node under which we write our data */
private final Node node;
- /** Counter used to build the name of Property to
- * which we currently write */
- private int counter;
+ /** Computes path for stream storage */
+ private final NodeStreamPath streamPath;
/** Buffer to hold data before writing it to a Property */
private final ByteArrayOutputStream buffer = new
ByteArrayOutputStream(BUFFER_SIZE);
@@ -65,6 +68,7 @@ public class NodeOutputStream extends Ou
public NodeOutputStream(Node n) {
node = n;
+ streamPath = new NodeStreamPath();
}
/** Calls flush to persist our stream, before closing */
@@ -79,15 +83,22 @@ public class NodeOutputStream extends Ou
*/
@Override
public void flush() throws IOException {
- counter++;
- // TODO use hierarchy to allow for arbitrary number of flush calls
- final String name = NodeOutputStream.STREAM_PROPERTY_NAME_PREFIX +
counter;
+
+ streamPath.selectNextPath();
+
try {
if(!node.getSession().isLive()) {
log.warn("Session closed, unable to flush stream");
} else {
- node.setProperty(name, new
ByteArrayInputStream(buffer.toByteArray()));
- log.debug("Saved {} bytes to Property {}", buffer.size(),
node.getProperty(name).getPath());
+ // Create node that will store the stream
+ final String streamNodePath = node.getPath() + "/" +
streamPath.getNodePath();
+ final Node streamNode = new
DeepNodeCreator().deepCreateNode(streamNodePath,
+ node.getSession(), STREAM_NODE_TYPE);
+
+ streamNode.setProperty(NodeStreamPath.PROPERTY_NAME,
+ new ByteArrayInputStream(buffer.toByteArray()));
+ log.debug("Saved {} bytes to Property {}", buffer.size(),
+
streamNode.getProperty(NodeStreamPath.PROPERTY_NAME).getPath());
node.save();
buffer.reset();
}
Added:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
URL:
http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java?rev=979260&view=auto
==============================================================================
---
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
(added)
+++
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
Mon Jul 26 12:21:43 2010
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sling.bgservlets.impl.nodestream;
+
+
+/** Builds sequential hierachical paths to store node
+ * streams: increment a counter and build path using
+ * CHARS_PER_LEVEL characters of the String value of
+ * the counter per path level.
+ */
+class NodeStreamPath {
+ private int counter;
+ private String path;
+ private final int CHARS_PER_LEVEL = 2;
+
+ /** Property name to use for streams */
+ static final String PROPERTY_NAME = "stream";
+
+ /** Select the next path to use, must be
+ * called before using getPath().
+ * Not thread-safe. */
+ void selectNextPath() {
+ counter++;
+ final StringBuilder sb = new StringBuilder();
+ final String str = String.valueOf(counter);
+ for(int i = 0; i < str.length(); i++) {
+ if(i> 0 && i % CHARS_PER_LEVEL == 0) {
+ sb.append('/');
+ }
+ sb.append(str.charAt(i));
+ }
+ path = sb.toString();
+ }
+
+ /** Return the last path computed by selectNextPath() */
+ String getNodePath() {
+ return path;
+ }
+}
Propchange:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
sling/trunk/contrib/extensions/bgservlets/src/main/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPath.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision Rev URL
Added:
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
URL:
http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java?rev=979260&view=auto
==============================================================================
---
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
(added)
+++
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
Mon Jul 26 12:21:43 2010
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sling.bgservlets.impl.nodestream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import org.junit.Test;
+
+public class NodeStreamPathTest {
+
+ @Test
+ public void testNullOnFirstCall() {
+ final NodeStreamPath nsp = new NodeStreamPath();
+ assertNull(nsp.getNodePath());
+ }
+
+ @Test
+ public void testPaths() {
+ final String [] data = {
+ "1", "1",
+ "9", "9",
+ "99", "99",
+ "100", "10/0",
+ "199", "19/9",
+ "200", "20/0",
+ "1234", "12/34",
+ };
+
+ for(int i=0; i < data.length; i += 2) {
+ final NodeStreamPath nsp = new NodeStreamPath();
+ for(int j = 0; j < Integer.parseInt(data[i]); j++) {
+ nsp.selectNextPath();
+ }
+ final String exp = data[i+1];
+ assertEquals("At index " + i + ", expected " + exp, exp,
nsp.getNodePath());
+ }
+ }
+}
Propchange:
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange:
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamPathTest.java
------------------------------------------------------------------------------
svn:keywords = Author Date Id Revision Rev URL
Modified:
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
URL:
http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java?rev=979260&r1=979259&r2=979260&view=diff
==============================================================================
---
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
(original)
+++
sling/trunk/contrib/extensions/bgservlets/src/test/java/org/apache/sling/bgservlets/impl/nodestream/NodeStreamTest.java
Mon Jul 26 12:21:43 2010
@@ -23,6 +23,8 @@ import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.RepositoryException;
import org.apache.sling.commons.testing.jcr.RepositoryTestBase;
@@ -97,14 +99,28 @@ public class NodeStreamTest extends Repo
nos.close();
assertFalse("Expecting no pending changes in testNode session",
testNode.getSession().hasPendingChanges());
- final long propCount = testNode.getProperties().getSize();
- final long expect = 10;
- assertTrue("Expecting > " + expect + " properties on test node",
propCount > expect);
+
+ // Stream must be stored in a hierarchy under testNode, to
+ // avoid limitations if flush() is called many times
+ final int childCount = getChildCount(testNode);
+ final int expect = 10;
+ assertTrue("Expecting > " + expect + " child nodes under testNode, got
" + childCount, childCount > expect);
final NodeInputStream nis = new NodeInputStream(testNode);
assertStream(new ByteArrayInputStream(BIG_DATA), nis);
}
+ private int getChildCount(Node n) throws RepositoryException {
+ int result = 0;
+ final NodeIterator it = n.getNodes();
+ while(it.hasNext()) {
+ result++;
+ final Node kid = it.nextNode();
+ result += getChildCount(kid);
+ }
+ return result;
+ }
+
public void testWriteWithOffset() throws Exception {
final Node testNode = getTestRootNode().addNode(NAME_PREFIX +
counter++);
testNode.getSession().save();
