Author: mduerig
Date: Thu Feb 12 09:44:46 2015
New Revision: 1659183
URL: http://svn.apache.org/r1659183
Log:
OAK-2504: oak-run debug should list a breakdown of space usage per record type
Initial implementation
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordType.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordType.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordType.java?rev=1659183&r1=1659182&r2=1659183&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordType.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordType.java
Thu Feb 12 09:44:46 2015
@@ -19,7 +19,7 @@ package org.apache.jackrabbit.oak.plugin
/**
* The type of a record in a segment.
*/
-enum RecordType {
+public enum RecordType {
/**
* A leaf of a map (which is a HAMT tree). This contains
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java?rev=1659183&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyser.java
Thu Feb 12 09:44:46 2015
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static org.apache.jackrabbit.oak.plugins.segment.ListRecord.LEVEL_SIZE;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.MEDIUM_LIMIT;
+import static
org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ID_BYTES;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.SMALL_LIMIT;
+import static
org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE;
+import static
org.apache.jackrabbit.oak.plugins.segment.Template.MANY_CHILD_NODES;
+import static
org.apache.jackrabbit.oak.plugins.segment.Template.ZERO_CHILD_NODES;
+
+import java.util.Formatter;
+import java.util.Set;
+
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+
+/**
+ * This utility breaks down space usage per record type.
+ * It accounts for value sharing. That is, an instance
+ * of this class will remember which records it has seen
+ * already and not count those again. Only the effective
+ * space taken by the records is taken into account. Slack
+ * space from aligning records is not accounted for.
+ */
+public class RecordUsageAnalyser {
+ private final Set<RecordId> seenIds = newHashSet();
+
+ private int mapSize; // leaf and branch
+ private int listSize; // list and bucket
+ private int valueSize; // inlined values
+ private int templateSize; // template
+ private int nodeSize; // node
+
+ public int getMapSize() {
+ return mapSize;
+ }
+
+ public int getListSize() {
+ return listSize;
+ }
+
+ public int getValueSize() {
+ return valueSize;
+ }
+
+ public int getTemplateSize() {
+ return templateSize;
+ }
+
+ public int getNodeSize() {
+ return nodeSize;
+ }
+
+ public void analyseNode(RecordId nodeId) {
+ if (seenIds.add(nodeId)) {
+ Segment segment = nodeId.getSegment();
+ int offset = nodeId.getOffset();
+ RecordId templateId = segment.readRecordId(offset);
+ analyseTemplate(templateId);
+
+ Template template = segment.readTemplate(templateId);
+
+ // Recurses into child nodes in this segment
+ if (template.getChildName() == MANY_CHILD_NODES) {
+ RecordId childMapId = segment.readRecordId(offset +
RECORD_ID_BYTES);
+ MapRecord childMap = segment.readMap(childMapId);
+ analyseMap(childMapId, childMap);
+ for (ChildNodeEntry childNodeEntry : childMap.getEntries()) {
+ NodeState child = childNodeEntry.getNodeState();
+ if (child instanceof SegmentNodeState) {
+ RecordId childId = ((SegmentNodeState)
child).getRecordId();
+ analyseNode(childId);
+ }
+ }
+ } else if (template.getChildName() != ZERO_CHILD_NODES) {
+ RecordId childId = segment.readRecordId(offset +
RECORD_ID_BYTES);
+ analyseNode(childId);
+ }
+
+ // Recurse into properties
+ int ids = template.getChildName() == ZERO_CHILD_NODES ? 1 : 2;
+ nodeSize += ids * RECORD_ID_BYTES;
+ PropertyTemplate[] propertyTemplates =
template.getPropertyTemplates();
+ for (PropertyTemplate propertyTemplate : propertyTemplates) {
+ nodeSize += RECORD_ID_BYTES;
+ RecordId propertyId = segment.readRecordId(offset + ids++ *
RECORD_ID_BYTES);
+ analyseProperty(propertyId, propertyTemplate);
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ Formatter formatter = new Formatter(sb);
+ formatter.format(
+ "%s in maps (leaf and branch records)%n",
+ byteCountToDisplaySize(mapSize));
+ formatter.format(
+ "%s in lists (list and bucket records)%n",
+ byteCountToDisplaySize(listSize));
+ formatter.format(
+ "%s in values (value and block records)%n",
+ byteCountToDisplaySize(valueSize));
+ formatter.format(
+ "%s in templates (template records)%n",
+ byteCountToDisplaySize(templateSize));
+ formatter.format(
+ "%s in nodes (node records)%n",
+ byteCountToDisplaySize(nodeSize));
+ return sb.toString();
+ }
+
+ private void analyseTemplate(RecordId templateId) {
+ if (seenIds.add(templateId)) {
+ Segment segment = templateId.getSegment();
+ int size = 0;
+ int offset = templateId.getOffset();
+ int head = segment.readInt(offset + size);
+ boolean hasPrimaryType = (head & (1 << 31)) != 0;
+ boolean hasMixinTypes = (head & (1 << 30)) != 0;
+ boolean zeroChildNodes = (head & (1 << 29)) != 0;
+ boolean manyChildNodes = (head & (1 << 28)) != 0;
+ int mixinCount = (head >> 18) & ((1 << 10) - 1);
+ int propertyCount = head & ((1 << 18) - 1);
+ size += 4;
+
+ if (hasPrimaryType) {
+ RecordId primaryId = segment.readRecordId(offset + size);
+ analyseString(primaryId);
+ size += Segment.RECORD_ID_BYTES;
+ }
+
+ if (hasMixinTypes) {
+ for (int i = 0; i < mixinCount; i++) {
+ RecordId mixinId = segment.readRecordId(offset + size);
+ analyseString(mixinId);
+ size += Segment.RECORD_ID_BYTES;
+ }
+ }
+
+ if (!zeroChildNodes && !manyChildNodes) {
+ RecordId childNameId = segment.readRecordId(offset + size);
+ analyseString(childNameId);
+ size += Segment.RECORD_ID_BYTES;
+ }
+
+ for (int i = 0; i < propertyCount; i++) {
+ RecordId propertyNameId = segment.readRecordId(offset + size);
+ size += Segment.RECORD_ID_BYTES;
+ size++; // type
+ analyseString(propertyNameId);
+ }
+ templateSize += size;
+ }
+ }
+
+ private void analyseMap(RecordId mapId, MapRecord map) {
+ if (seenIds.add(mapId)) {
+ if (map.isDiff()) {
+ analyseDiff(mapId, map);
+ } else if (map.isLeaf()) {
+ analyseLeaf(map);
+ } else {
+ analyseBranch(map);
+ }
+ }
+ }
+
+ private void analyseDiff(RecordId mapId, MapRecord map) {
+ mapSize += 4; // -1
+ mapSize += 4; // hash of changed key
+ mapSize += RECORD_ID_BYTES; // key
+ mapSize += RECORD_ID_BYTES; // value
+ mapSize += RECORD_ID_BYTES; // base
+
+ RecordId baseId = mapId.getSegment().readRecordId(
+ mapId.getOffset() + 8 + 2 * RECORD_ID_BYTES);
+ analyseMap(baseId, new MapRecord(baseId));
+ }
+
+ private void analyseLeaf(MapRecord map) {
+ mapSize += 4; // size
+ mapSize += map.size() * 4; // key hashes
+
+ for (MapEntry entry : map.getEntries()) {
+ mapSize += 2 * RECORD_ID_BYTES; // key value pairs
+ analyseString(entry.getKey());
+ }
+ }
+
+ private void analyseBranch(MapRecord map) {
+ mapSize += 4; // level/size
+ mapSize += 4; // bitmap
+ for (MapRecord bucket : map.getBuckets()) {
+ if (bucket != null) {
+ mapSize += RECORD_ID_BYTES;
+ analyseMap(bucket.getRecordId(), bucket);
+ }
+ }
+ }
+
+ private void analyseProperty(RecordId propertyId, PropertyTemplate
template) {
+ if (!seenIds.contains(propertyId)) {
+ Segment segment = propertyId.getSegment();
+ int offset = propertyId.getOffset();
+ Type<?> type = template.getType();
+
+ if (type.isArray()) {
+ seenIds.add(propertyId);
+ int size = segment.readInt(offset);
+ valueSize += 4;
+
+ if (size > 0) {
+ RecordId listId = segment.readRecordId(offset + 4);
+ valueSize += RECORD_ID_BYTES;
+ for (RecordId valueId : new ListRecord(listId,
size).getEntries()) {
+ analyseValue(valueId, type.getBaseType());
+ }
+ analyseList(listId, size);
+ }
+ } else {
+ analyseValue(propertyId, type);
+ }
+ }
+ }
+
+ private void analyseValue(RecordId valueId, Type<?> type) {
+ checkArgument(!type.isArray());
+ if (type == BINARY) {
+ analyseBlob(valueId);
+ } else {
+ analyseString(valueId);
+ }
+ }
+
+ private void analyseBlob(RecordId blobId) {
+ if (seenIds.add(blobId)) {
+ Segment segment = blobId.getSegment();
+ int offset = blobId.getOffset();
+ byte head = segment.readByte(offset);
+ if ((head & 0x80) == 0x00) {
+ // 0xxx xxxx: small value
+ valueSize += (1 + head);
+ } else if ((head & 0xc0) == 0x80) {
+ // 10xx xxxx: medium value
+ int length = (segment.readShort(offset) & 0x3fff) +
SMALL_LIMIT;
+ valueSize += (2 + length);
+ } else if ((head & 0xe0) == 0xc0) {
+ // 110x xxxx: long value
+ long length = (segment.readLong(offset) & 0x1fffffffffffffffL)
+ MEDIUM_LIMIT;
+ int size = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE);
+ RecordId listId = segment.readRecordId(offset + 8);
+ analyseList(listId, size);
+ valueSize += (8 + RECORD_ID_BYTES + length);
+ } else if ((head & 0xf0) == 0xe0) {
+ // 1110 xxxx: external value
+ int length = (head & 0x0f) << 8 | (segment.readByte(offset +
1) & 0xff);
+ valueSize += (2 + length);
+ } else {
+ throw new IllegalStateException(String.format(
+ "Unexpected value record type: %02x", head & 0xff));
+ }
+ }
+ }
+
+ private void analyseString(RecordId stringId) {
+ if (seenIds.add(stringId)) {
+ Segment segment = stringId.getSegment();
+ int offset = stringId.getOffset();
+
+ long length = segment.readLength(offset);
+ if (length < Segment.SMALL_LIMIT) {
+ valueSize += (1 + length);
+ } else if (length < Segment.MEDIUM_LIMIT) {
+ valueSize += (2 + length);
+ } else if (length < Integer.MAX_VALUE) {
+ int size = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE);
+ RecordId listId = segment.readRecordId(offset + 8);
+ analyseList(listId, size);
+ valueSize += (8 + RECORD_ID_BYTES + length);
+ } else {
+ throw new IllegalStateException("String is too long: " +
length);
+ }
+ }
+ }
+
+ private void analyseList(RecordId listId, int size) {
+ if (seenIds.add(listId)) {
+ listSize += noOfListSlots(size) * RECORD_ID_BYTES;
+ }
+ }
+
+ private static int noOfListSlots(int size) {
+ if (size <= LEVEL_SIZE) {
+ return size;
+ } else {
+ int fullBuckets = size / LEVEL_SIZE;
+ if (size % LEVEL_SIZE > 1) {
+ return size + noOfListSlots(fullBuckets + 1);
+ } else {
+ return size + noOfListSlots(fullBuckets);
+ }
+ }
+ }
+
+}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1659183&r1=1659182&r2=1659183&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
Thu Feb 12 09:44:46 2015
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkPositionIndexes;
import static com.google.common.base.Preconditions.checkState;
@@ -34,13 +35,12 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.base.Charsets;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
-import com.google.common.base.Charsets;
-
/**
* A list of records.
* <p>
@@ -187,6 +187,23 @@ public class Segment {
return (data.get(REF_COUNT_OFFSET) & 0xff) + 1;
}
+ public int getRootCount() {
+ return data.getShort(ROOT_COUNT_OFFSET) & 0xffff;
+ }
+
+ public RecordType getRootType(int index) {
+ int refCount = getRefCount();
+ checkArgument(index < getRootCount());
+ return RecordType.values()[data.get(data.position() + refCount * 16 +
index * 3) & 0xff];
+ }
+
+ public int getRootOffset(int index) {
+ int refCount = getRefCount();
+ checkArgument(index < getRootCount());
+ return (data.getShort(data.position() + refCount * 16 + index * 3 + 1)
& 0xffff)
+ << RECORD_ALIGN_BITS;
+ }
+
SegmentId getRefId(int index) {
if (refids == null || index >= refids.length) {
String type = "data";
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java?rev=1659183&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordUsageAnalyserTest.java
Thu Feb 12 09:44:46 2015
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static java.util.Collections.nCopies;
+import static org.apache.jackrabbit.oak.api.Type.LONGS;
+import static org.apache.jackrabbit.oak.api.Type.NAME;
+import static org.apache.jackrabbit.oak.api.Type.NAMES;
+import static org.apache.jackrabbit.oak.api.Type.STRINGS;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.apache.jackrabbit.oak.plugins.segment.ListRecord.LEVEL_SIZE;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.MEDIUM_LIMIT;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.SMALL_LIMIT;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Random;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Before;
+import org.junit.Test;
+
+public class RecordUsageAnalyserTest {
+ private SegmentStore store;
+ private SegmentWriter writer;
+ private RecordUsageAnalyser analyser = new RecordUsageAnalyser();
+
+ @Before
+ public void setup() {
+ store = mock(SegmentStore.class);
+ SegmentTracker tracker = new SegmentTracker(store);
+ when(store.getTracker()).thenReturn(tracker);
+ writer = new SegmentWriter(store, store.getTracker());
+ analyser = new RecordUsageAnalyser();
+ }
+
+ @Test
+ public void emptyNode() {
+ SegmentNodeState node = writer.writeNode(EMPTY_NODE);
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 0, 4, 3);
+ }
+
+ @Test
+ public void nodeWithInt() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("one", 1);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 6, 8, 6);
+ }
+
+ @Test
+ public void nodeWithString() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("two", "222");
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 8, 8, 6);
+ }
+
+ @Test
+ public void nodeWithMediumString() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("medium", Strings.repeat("a", SMALL_LIMIT + 1));
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 138, 8, 6);
+ }
+
+ @Test
+ public void nodeWithLargeString() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("large", Strings.repeat("b", MEDIUM_LIMIT + 1));
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 15, 16530, 8, 6);
+ }
+
+ @Test
+ public void nodeWithSameString() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("two", "two");
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 4, 8, 6);
+ }
+
+ @Test
+ public void nodeWithInts() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("multi", ImmutableList.of(1L, 2L, 3L, 4L), LONGS);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 12, 21, 8, 6);
+ }
+
+ @Test
+ public void nodeWithManyInts() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("multi", nCopies(LEVEL_SIZE + 1, 1L), LONGS);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 771, 15, 8, 6);
+ }
+
+ @Test
+ public void nodeWithManyIntsAndOne() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("multi", nCopies(LEVEL_SIZE + 2, 1L), LONGS);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 777, 15, 8, 6);
+ }
+
+ @Test
+ public void nodeWithStrings() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("multi", ImmutableList.of("one", "one", "two",
"two", "three"), STRINGS);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 15, 27, 8, 6);
+ }
+
+ @Test
+ public void nodeWithBlob() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("blob", new ArrayBasedBlob(new byte[]{1, 2, 3,
4}));
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 10, 8, 6);
+ }
+
+ @Test
+ public void nodeWithMediumBlob() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+
+ byte[] bytes = new byte[SMALL_LIMIT + 1];
+ new Random().nextBytes(bytes);
+ builder.setProperty("mediumBlob", new ArrayBasedBlob(bytes));
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 142, 8, 6);
+ }
+
+ @Test
+ public void nodeWithLargeBlob() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+
+ byte[] bytes = new byte[MEDIUM_LIMIT + 1];
+ new Random().nextBytes(bytes);
+ builder.setProperty("largeBlob", new ArrayBasedBlob(bytes));
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 15, 16534, 8, 6);
+ }
+
+ @Test
+ public void nodeWithPrimaryType() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("jcr:primaryType", "type", NAME);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 5, 7, 3);
+ }
+
+ @Test
+ public void nodeWithMixinTypes() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty("jcr:mixinTypes", ImmutableList.of("type1",
"type2"), NAMES);
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 12, 10, 3);
+ }
+
+ @Test
+ public void singleChild() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setChildNode("child");
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 0, 0, 6, 11, 9);
+ }
+
+ @Test
+ public void multiChild() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setChildNode("child1");
+ builder.setChildNode("child2");
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 24, 0, 14, 8, 12);
+ }
+
+ @Test
+ public void manyChild() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ for (int k = 0; k < MapRecord.BUCKETS_PER_LEVEL + 1; k++) {
+ builder.setChildNode("child" + k);
+ }
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 457, 0, 254, 8, 105);
+ }
+
+ @Test
+ public void changedChild() {
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setChildNode("child1");
+ builder.setChildNode("child2");
+
+ SegmentNodeState node = writer.writeNode(builder.getNodeState());
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 24, 0, 14, 8, 12);
+
+ builder = node.builder();
+ builder.child("child1").setProperty("p", "q");
+
+
when(store.containsSegment(node.getRecordId().getSegmentId())).thenReturn(true);
+ node = (SegmentNodeState) builder.getNodeState();
+
+ analyser.analyseNode(node.getRecordId());
+ assertSizes(analyser, 41, 0, 18, 16, 24);
+ }
+
+ private static void assertSizes(RecordUsageAnalyser analyser,
+ int maps, int lists, int values, int templates, int nodes) {
+ assertEquals("maps sizes mismatch", maps, analyser.getMapSize());
+ assertEquals("lists sizes mismatch", lists, analyser.getListSize());
+ assertEquals("value sizes mismatch", values, analyser.getValueSize());
+ assertEquals("template sizes mismatch", templates,
analyser.getTemplateSize());
+ assertEquals("nodes sizes mismatch", nodes, analyser.getNodeSize());
+ }
+
+}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1659183&r1=1659182&r2=1659183&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Thu Feb 12 09:44:46 2015
@@ -18,7 +18,9 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Arrays.asList;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.checkpoint.Checkpoints.CP;
+import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
import static
org.apache.jackrabbit.oak.plugins.segment.file.tooling.ConsistencyChecker.checkConsistency;
import java.io.Closeable;
@@ -56,12 +58,10 @@ import com.google.common.util.concurrent
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoURI;
-
import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
-
import org.apache.jackrabbit.core.RepositoryContext;
import org.apache.jackrabbit.core.config.RepositoryConfig;
import org.apache.jackrabbit.oak.Oak;
@@ -90,6 +90,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.document.util.MapFactory;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
+import org.apache.jackrabbit.oak.plugins.segment.RecordUsageAnalyser;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
@@ -805,20 +806,21 @@ public class Main {
}
}
- private static void debugFileStore(FileStore store){
-
+ private static void debugFileStore(FileStore store) {
Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
-
int dataCount = 0;
long dataSize = 0;
int bulkCount = 0;
long bulkSize = 0;
+ RecordUsageAnalyser analyser = new RecordUsageAnalyser();
+
for (SegmentId id : store.getSegmentIds()) {
if (id.isDataSegmentId()) {
Segment segment = id.getSegment();
dataCount++;
dataSize += segment.size();
idmap.put(id, segment.getReferencedIds());
+ analyseSegment(segment, analyser);
} else if (id.isBulkSegmentId()) {
bulkCount++;
bulkSize += id.getSegment().size();
@@ -827,11 +829,12 @@ public class Main {
}
System.out.println("Total size:");
System.out.format(
- "%6dMB in %6d data segments%n",
- dataSize / (1024 * 1024), dataCount);
+ "%s in %6d data segments%n",
+ byteCountToDisplaySize(dataSize), dataCount);
System.out.format(
- "%6dMB in %6d bulk segments%n",
- bulkSize / (1024 * 1024), bulkCount);
+ "%s in %6d bulk segments%n",
+ byteCountToDisplaySize(bulkSize), bulkCount);
+ System.out.println(analyser.toString());
Set<SegmentId> garbage = newHashSet(idmap.keySet());
Queue<SegmentId> queue = Queues.newArrayDeque();
@@ -855,13 +858,22 @@ public class Main {
bulkSize += id.getSegment().size();
}
}
- System.out.println("Available for garbage collection:");
+ System.out.println("\nAvailable for garbage collection:");
System.out.format(
- "%6dMB in %6d data segments%n",
- dataSize / (1024 * 1024), dataCount);
+ "%s in %6d data segments%n",
+ byteCountToDisplaySize(dataSize), dataCount);
System.out.format(
- "%6dMB in %6d bulk segments%n",
- bulkSize / (1024 * 1024), bulkCount);
+ "%s in %6d bulk segments%n",
+ byteCountToDisplaySize(bulkSize), bulkCount);
+ }
+
+ private static void analyseSegment(Segment segment, RecordUsageAnalyser
analyser) {
+ for (int k = 0; k < segment.getRootCount(); k++) {
+ if (segment.getRootType(k) == NODE) {
+ RecordId nodeId = new RecordId(segment.getSegmentId(),
segment.getRootOffset(k));
+ analyser.analyseNode(nodeId);
+ }
+ }
}
/**
