Author: mreutegg
Date: Mon Oct 5 09:08:16 2015
New Revision: 1706769
URL: http://svn.apache.org/viewvc?rev=1706769&view=rev
Log:
OAK-3455: Improve conflict exception message
CommitFailedException may not have the message, but the DocumentStoreException
cause must have the it
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStoreHelper.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java?rev=1706769&r1=1706768&r2=1706769&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java
Mon Oct 5 09:08:16 2015
@@ -1613,7 +1613,8 @@ public class DocumentNodeStoreTest {
merge(ns2, b2);
fail("Must throw CommitFailedException");
} catch (CommitFailedException e) {
- assertTrue(e.getMessage().contains("not yet visible"));
+ assertNotNull(e.getCause());
+ assertTrue(e.getCause().getMessage().contains("not yet visible"));
}
}
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java?rev=1706769&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
Mon Oct 5 09:08:16 2015
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
+import com.mongodb.DBObject;
+import com.mongodb.QueryBuilder;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.json.JsopReader;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
+import
org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStoreHelper;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+
+import static
org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStoreHelper.convertFromDBObject;
+
+/**
+ * Helper class to access package private method of DocumentNodeStore and other
+ * classes in this package.
+ */
+public class DocumentNodeStoreHelper {
+
+ public static void garbageReport(DocumentNodeStore dns) {
+ System.out.print("Collecting top 100 nodes with most blob garbage ");
+ Stopwatch sw = Stopwatch.createStarted();
+ Iterable<BlobReferences> refs = scan(dns, new
BlobGarbageSizeComparator(), 100);
+ for (BlobReferences br : refs) {
+ System.out.println(br);
+ }
+ System.out.println("Collected in " + sw.stop());
+ }
+
+ private static Iterable<BlobReferences> scan(DocumentNodeStore store,
+ Comparator<BlobReferences>
comparator,
+ int num) {
+ long totalGarbage = 0;
+ Iterable<NodeDocument> docs = getDocuments(store.getDocumentStore());
+ PriorityQueue<BlobReferences> queue = new
PriorityQueue<BlobReferences>(num, comparator);
+ List<Blob> blobs = Lists.newArrayList();
+ long docCount = 0;
+ for (NodeDocument doc : docs) {
+ if (++docCount % 10000 == 0) {
+ System.out.print(".");
+ }
+ blobs.clear();
+ BlobReferences refs = collectReferences(doc, store);
+ totalGarbage += refs.garbageSize;
+ queue.add(refs);
+ if (queue.size() > num) {
+ queue.remove();
+ }
+ }
+
+ System.out.println();
+ List<BlobReferences> refs = Lists.newArrayList();
+ refs.addAll(queue);
+ Collections.sort(refs, Collections.reverseOrder(comparator));
+ System.out.println("Total garbage size: " +
FileUtils.byteCountToDisplaySize(totalGarbage));
+ System.out.println("Total number of nodes with blob references: " +
docCount);
+ System.out.println("total referenced / old referenced / # blob
references / path");
+ return refs;
+ }
+
+ private static BlobReferences collectReferences(NodeDocument doc,
+ DocumentNodeStore ns) {
+ long blobSize = 0;
+ long garbageSize = 0;
+ int numBlobs = 0;
+
+ List<Blob> blobs = Lists.newArrayList();
+ Revision head = ns.getHeadRevision();
+ boolean exists = doc.getNodeAtRevision(ns, head, null) != null;
+ for (String key : doc.keySet()) {
+ if (!Utils.isPropertyName(key)) {
+ continue;
+ }
+ boolean foundValid = false;
+ Map<Revision, String> valueMap = doc.getLocalMap(key);
+ for (Map.Entry<Revision, String> entry : valueMap.entrySet()) {
+ blobs.clear();
+ String v = entry.getValue();
+ if (v != null) {
+ loadValue(v, blobs, ns);
+ }
+ blobSize += size(blobs);
+ if (foundValid) {
+ garbageSize += size(blobs);
+ } else if (doc.isCommitted(entry.getKey())) {
+ foundValid = true;
+ } else {
+ garbageSize += size(blobs);
+ }
+ numBlobs += blobs.size();
+ }
+ }
+ return new BlobReferences(doc.getPath(), blobSize, numBlobs,
garbageSize, exists);
+ }
+
+ private static Iterable<NodeDocument> getDocuments(DocumentStore store) {
+ if (store instanceof MongoDocumentStore) {
+ // optimized implementation for MongoDocumentStore
+ final MongoDocumentStore mds = (MongoDocumentStore) store;
+ DBCollection dbCol = MongoDocumentStoreHelper.getDBCollection(
+ mds, Collection.NODES);
+ DBObject query = QueryBuilder.start(NodeDocument.HAS_BINARY_FLAG)
+ .is(NodeDocument.HAS_BINARY_VAL)
+ .get();
+ DBCursor cursor = dbCol.find(query);
+ return Iterables.transform(cursor, new Function<DBObject,
NodeDocument>() {
+ @Nullable
+ @Override
+ public NodeDocument apply(DBObject input) {
+ return convertFromDBObject(mds, Collection.NODES, input);
+ }
+ });
+ } else {
+ return Utils.getSelectedDocuments(store,
+ NodeDocument.HAS_BINARY_FLAG, NodeDocument.HAS_BINARY_VAL);
+ }
+ }
+
+
+ private static long size(Iterable<Blob> blobs) {
+ long size = 0;
+ for (Blob b : blobs) {
+ size += b.length();
+ }
+ return size;
+ }
+
+ private static void loadValue(String v, java.util.Collection<Blob> blobs,
+ DocumentNodeStore nodeStore) {
+ JsopReader reader = new JsopTokenizer(v);
+ PropertyState p;
+ if (reader.matches('[')) {
+ p = DocumentPropertyState.readArrayProperty("x", nodeStore,
reader);
+ if (p.getType() == Type.BINARIES) {
+ for (int i = 0; i < p.count(); i++) {
+ Blob b = p.getValue(Type.BINARY, i);
+ blobs.add(b);
+ }
+ }
+ } else {
+ p = DocumentPropertyState.readProperty("x", nodeStore, reader);
+ if (p.getType() == Type.BINARY) {
+ Blob b = p.getValue(Type.BINARY);
+ blobs.add(b);
+ }
+ }
+ }
+
+ private static class BlobReferences {
+
+ final String path;
+ final long blobSize;
+ final long garbageSize;
+ final int numBlobs;
+ final boolean exists;
+
+ public BlobReferences(String path,
+ long blobSize,
+ int numBlobs,
+ long garbageSize,
+ boolean exists) {
+ this.path = path;
+ this.blobSize = blobSize;
+ this.garbageSize = garbageSize;
+ this.numBlobs = numBlobs;
+ this.exists = exists;
+ }
+
+ @Override
+ public String toString() {
+ String s = FileUtils.byteCountToDisplaySize(blobSize) + "\t"
+ + FileUtils.byteCountToDisplaySize(garbageSize) + "\t"
+ + numBlobs + "\t"
+ + path;
+ if (!exists) {
+ s += "\t(deleted)";
+ }
+ return s;
+ }
+ }
+
+ private static class BlobGarbageSizeComparator
+ implements Comparator<BlobReferences> {
+
+ @Override
+ public int compare(BlobReferences o1, BlobReferences o2) {
+ int c = Long.compare(o1.garbageSize, o2.garbageSize);
+ if (c != 0) {
+ return c;
+ }
+ return o1.path.compareTo(o2.path);
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStoreHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStoreHelper.java?rev=1706769&r1=1706768&r2=1706769&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStoreHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStoreHelper.java
Mon Oct 5 09:08:16 2015
@@ -27,6 +27,7 @@ import com.mongodb.DBObject;
import com.mongodb.QueryBuilder;
import com.mongodb.WriteResult;
+import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.Document;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.Revision;
@@ -92,4 +93,14 @@ public class MongoDocumentStoreHelper {
}
+ public static <T extends Document> DBCollection getDBCollection(
+ MongoDocumentStore store, Collection<T> c) {
+ return store.getDBCollection(c);
+ }
+
+ public static <T extends Document> T convertFromDBObject(
+ MongoDocumentStore store, Collection<T> col, DBObject obj) {
+ return store.convertFromDBObject(col, obj);
+ }
+
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1706769&r1=1706768&r2=1706769&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Mon Oct 5 09:08:16 2015
@@ -81,6 +81,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.backup.FileStoreRestore;
import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper;
import org.apache.jackrabbit.oak.plugins.document.LastRevRecoveryAgent;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
@@ -194,6 +195,9 @@ public final class Main {
case TIKA:
TextExtractorMain.main(args);
break;
+ case GARBAGE:
+ garbage(args);
+ break;
case HELP:
default:
System.err.print("Available run modes: ");
@@ -422,6 +426,7 @@ public final class Main {
closer.register(asCloseable(mongo));
DocumentNodeStore store = new DocumentMK.Builder()
.setMongoDB(mongo.getDB())
+ .setLeaseCheck(false)
.setClusterId(clusterId.value(options)).getNodeStore();
closer.register(asCloseable(store));
return store;
@@ -698,6 +703,25 @@ public final class Main {
}
}
+ private static void garbage(String[] args) throws IOException {
+ Closer closer = Closer.create();
+ String h = "garbage mongodb://host:port/database";
+ try {
+ NodeStore store = bootstrapNodeStore(args, closer, h);
+ if (!(store instanceof DocumentNodeStore)) {
+ System.err.println("Garbage mode only available for
DocumentNodeStore");
+ System.exit(1);
+ }
+ DocumentNodeStore dns = (DocumentNodeStore) store;
+
+ DocumentNodeStoreHelper.garbageReport(dns);
+ } catch (Throwable e) {
+ throw closer.rethrow(e);
+ } finally {
+ closer.close();
+ }
+ }
+
private static void debug(String[] args) throws IOException {
if (args.length == 0) {
System.err.println("usage: debug <path> [id...]");
@@ -1150,7 +1174,8 @@ public final class Main {
CHECKPOINTS("checkpoints"),
RECOVERY("recovery"),
REPAIR("repair"),
- TIKA("tika");
+ TIKA("tika"),
+ GARBAGE("garbage");
private final String name;
