This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 575019f9a1 Issue/oak 10636 (#1296)
575019f9a1 is described below
commit 575019f9a1f00136eb5d571e6f2e7263b46f86db
Author: Miroslav Smiljanic <[email protected]>
AuthorDate: Fri Feb 23 15:08:13 2024 +0100
Issue/oak 10636 (#1296)
* OAK-10636 test and fist iteration for the solution
* OAK-10636 modify TarFiles builder
* OAK-10636 ignore .DS_Store
* OAK-10636 ignore move segment map outside of the loop
* OAK-10636 avoid code duplication
* OAK-10636 check if TarFiles is initialised
* OAK-10636 remove logger
---------
Co-authored-by: Miroslav Smiljanic <[email protected]>
---
.gitignore | 1 +
.../apache/jackrabbit/oak/segment/SegmentBlob.java | 31 ++++++-
.../oak/segment/file/AbstractFileStore.java | 6 +-
.../jackrabbit/oak/segment/file/FileStore.java | 8 +-
.../oak/segment/file/ReadOnlyFileStore.java | 3 +-
.../oak/segment/file/tar/EntryRecovery.java | 9 ++
.../jackrabbit/oak/segment/file/tar/TarFiles.java | 56 ++++++++++--
.../jackrabbit/oak/segment/file/tar/TarReader.java | 18 +++-
.../jackrabbit/oak/segment/IdMappingBlobStore.java | 88 ++++++++++++++++++
.../oak/segment/LongIdMappingBlobStore.java} | 16 ++--
.../jackrabbit/oak/segment/file/FileStoreTest.java | 100 ++++++++++++++++++---
.../oak/segment/file/tar/TarFilesTest.java | 55 ++++++++++--
12 files changed, 347 insertions(+), 44 deletions(-)
diff --git a/.gitignore b/.gitignore
index d13284858c..c700561aa7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ atlassian-ide-plugin.xml
derby.log
.java-version
oak-shaded-guava/dependency-reduced-pom.xml
+.DS_Store
\ No newline at end of file
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
index 9ab2ffea8d..05cca5caa3 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
@@ -27,7 +27,9 @@ import static
org.apache.jackrabbit.oak.segment.SegmentStream.BLOCK_SIZE;
import java.io.InputStream;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
@@ -164,19 +166,28 @@ public class SegmentBlob extends Record implements Blob {
}
@Nullable
- public static String readBlobId(@NotNull Segment segment, int
recordNumber) {
+ private static String readBlobId(@NotNull Segment segment, int
recordNumber, Function<Segment, String> readLongBlobIdFunction) {
byte head = segment.readByte(recordNumber);
if ((head & 0xf0) == 0xe0) {
// 1110 xxxx: external value, small blob ID
return readShortBlobId(segment, recordNumber, head);
} else if ((head & 0xf8) == 0xf0) {
// 1111 0xxx: external value, long blob ID
- return readLongBlobId(segment, recordNumber);
+ return readLongBlobIdFunction.apply(segment);
} else {
return null;
}
}
+ @Nullable
+ public static String readBlobId(@NotNull Segment segment, int
recordNumber) {
+ return readBlobId(segment, recordNumber, s -> readLongBlobId(s,
recordNumber));
+ }
+
+ @Nullable
+ public static String readBlobId(@NotNull Segment segment, int
recordNumber, Map<SegmentId, Segment> recoveredSegments) {
+ return readBlobId(segment, recordNumber, s -> readLongBlobId(s,
recordNumber, recoveredSegments));
+ }
//------------------------------------------------------------< Object >--
@Override
@@ -230,10 +241,22 @@ public class SegmentBlob extends Record implements Blob {
return segment.readBytes(recordNumber, 2,
length).decode(UTF_8).toString();
}
- private static String readLongBlobId(Segment segment, int recordNumber) {
+ private static String readLongBlobId(Segment segment, int recordNumber,
Function<RecordId, Segment> getSegmentFunction) {
RecordId blobId = segment.readRecordId(recordNumber, 1);
+ Segment blobIdSegment = getSegmentFunction.apply(blobId);
+
+ return blobIdSegment.readString(blobId.getRecordNumber());
+ }
+
+ private static String readLongBlobId(Segment segment, int recordNumber) {
+ return readLongBlobId(segment, recordNumber, RecordId::getSegment);
+ }
- return blobId.getSegment().readString(blobId.getRecordNumber());
+ private static String readLongBlobId(Segment segment, int recordNumber,
Map<SegmentId, Segment> recoveredSegments) {
+ return readLongBlobId(segment, recordNumber, recordId -> {
+ Segment blobIdSegment =
recoveredSegments.get(recordId.getSegmentId());
+ return blobIdSegment != null ? blobIdSegment :
recordId.getSegment();
+ });
}
private List<RecordId> getBulkRecordIds() {
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
index 61cea135d1..9f5e168653 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
@@ -232,7 +232,9 @@ public abstract class AbstractFileStore implements
SegmentStore, Closeable {
: GCGeneration.NULL;
w.recoverEntry(msb, lsb, data, 0, data.length, generation);
if (SegmentId.isDataSegmentId(lsb)) {
- Segment segment = new Segment(tracker, segmentReader,
tracker.newSegmentId(msb, lsb), buffer);
+ SegmentId segmentId = tracker.newSegmentId(msb, lsb);
+ Segment segment = new Segment(tracker, segmentReader, segmentId,
buffer);
+ w.addSegment(segment);
populateTarGraph(segment, w);
populateTarBinaryReferences(segment, w);
}
@@ -250,7 +252,7 @@ public abstract class AbstractFileStore implements
SegmentStore, Closeable {
final UUID id = segment.getSegmentId().asUUID();
segment.forEachRecord((number, type, offset) -> {
if (type == RecordType.BLOB_ID) {
- w.recoverBinaryReference(generation, id,
SegmentBlob.readBlobId(segment, number));
+ w.recoverBinaryReference(generation, id,
SegmentBlob.readBlobId(segment, number, w.getRecoveredSegments()));
}
});
}
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
index 57fb74379b..6381e1129c 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
@@ -155,6 +155,8 @@ public class FileStore extends AbstractFileStore {
this.stats = new FileStoreStats(statsProvider, this, 0);
+ this.snfeListener = builder.getSnfeListener();
+
CounterStats readerCountStats =
statsProvider.getCounterStats(TAR_READER_COUNT, DEFAULT);
CounterStats segmentCountStats =
statsProvider.getCounterStats(SEGMENT_COUNT, DEFAULT);
TarFiles.Builder tarFilesBuilder = TarFiles.builder()
@@ -167,9 +169,12 @@ public class FileStore extends AbstractFileStore {
.withMaxFileSize(builder.getMaxFileSize() * MB)
.withPersistence(builder.getPersistence())
.withReaderCountStats(readerCountStats)
- .withSegmentCountStats(segmentCountStats);
+ .withSegmentCountStats(segmentCountStats)
+ .withInitialisedReadersAndWriters(false);
this.tarFiles = tarFilesBuilder.build();
+ this.tarFiles.init();
+
long size = this.tarFiles.size();
this.stats.init(size);
@@ -202,7 +207,6 @@ public class FileStore extends AbstractFileStore {
.build(this)
);
- this.snfeListener = builder.getSnfeListener();
this.eagerSegmentCaching = builder.getEagerSegmentCaching();
TimerStats flushTimer = statsProvider.getTimer("oak.segment.flush",
METRICS_ONLY);
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
index bf25f32b74..97b2dc7ef7 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
@@ -77,8 +77,9 @@ public class ReadOnlyFileStore extends AbstractFileStore {
.withMemoryMapping(memoryMapping)
.withReadOnly()
.withPersistence(builder.getPersistence())
+ .withInitialisedReadersAndWriters(false)
.build();
-
+ tarFiles.init();
writer =
defaultSegmentWriterBuilder("read-only").withoutCache().build(this);
gcRetainedGenerations =
builder.getGcOptions().getRetainedGenerations();
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
index 34bef73a9c..f32342581c 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
@@ -17,7 +17,11 @@
package org.apache.jackrabbit.oak.segment.file.tar;
+import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+
import java.io.IOException;
+import java.util.Map;
import java.util.UUID;
public interface EntryRecovery {
@@ -28,4 +32,9 @@ public interface EntryRecovery {
void recoverBinaryReference(GCGeneration generation, UUID segmentId,
String reference);
+ Segment getSegment(SegmentId id);
+
+ void addSegment(Segment segment);
+
+ Map<SegmentId, Segment> getRecoveredSegments();
}
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
index b3cb9fd06e..fbd48e15fc 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
@@ -49,6 +49,7 @@ import java.util.regex.Pattern;
import org.apache.jackrabbit.guava.common.base.Predicate;
import org.apache.jackrabbit.guava.common.collect.Iterables;
+import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.file.FileReaper;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor;
@@ -136,6 +137,8 @@ public class TarFiles implements Closeable {
private CounterStats segmentCountStats = NoopStats.INSTANCE;
+ private boolean initialiseReadersAndWriters = true;
+
private Builder() {
// Prevent external instantiation.
}
@@ -201,6 +204,11 @@ public class TarFiles implements Closeable {
return this;
}
+ public Builder withInitialisedReadersAndWriters(boolean
initialiseReadersAndWriters) {
+ this.initialiseReadersAndWriters = initialiseReadersAndWriters;
+ return this;
+ }
+
public TarFiles build() throws IOException {
checkState(directory != null, "Directory not specified");
checkState(tarRecovery != null, "TAR recovery strategy not
specified");
@@ -369,6 +377,15 @@ public class TarFiles implements Closeable {
*/
private final CounterStats segmentCount;
+ private final boolean readOnly;
+
+ private final TarRecovery tarRecovery;
+
+ /**
+ * If {@code true}, the readers and writers are initialised.
+ */
+ private boolean initialised;
+
private static int getSegmentCount(TarReader reader) {
return reader.getEntries().length;
}
@@ -378,7 +395,15 @@ public class TarFiles implements Closeable {
archiveManager = builder.buildArchiveManager();
readerCount = builder.readerCountStats;
segmentCount = builder.segmentCountStats;
+ readOnly = builder.readOnly;
+ tarRecovery = builder.tarRecovery;
+ if (builder.initialiseReadersAndWriters) {
+ init();
+ }
+ }
+
+ public void init() throws IOException {
Map<Integer, Map<Character, String>> map =
collectFiles(archiveManager);
Integer[] indices = map.keySet().toArray(new Integer[map.size()]);
Arrays.sort(indices);
@@ -390,23 +415,30 @@ public class TarFiles implements Closeable {
for (Integer index : indices) {
TarReader r;
- if (builder.readOnly) {
- r = TarReader.openRO(map.get(index), builder.tarRecovery,
archiveManager);
+ if (readOnly) {
+ r = TarReader.openRO(map.get(index), tarRecovery,
archiveManager);
} else {
- r = TarReader.open(map.get(index), builder.tarRecovery,
archiveManager);
+ r = TarReader.open(map.get(index), tarRecovery,
archiveManager);
}
segmentCount.inc(getSegmentCount(r));
readers = new Node(r, readers);
readerCount.inc();
}
- if (builder.readOnly) {
- return;
+ if (!readOnly) {
+ int writeNumber = 0;
+ if (indices.length > 0) {
+ writeNumber = indices[indices.length - 1] + 1;
+ }
+ writer = new TarWriter(archiveManager, writeNumber, segmentCount);
}
- int writeNumber = 0;
- if (indices.length > 0) {
- writeNumber = indices[indices.length - 1] + 1;
+
+ initialised = true;
+ }
+
+ private void checkInitialised() {
+ if (!initialised) {
+ throw new IllegalRepositoryStateException("TarFiles not
initialised");
}
- writer = new TarWriter(archiveManager, writeNumber, segmentCount);
}
@Override
@@ -530,6 +562,7 @@ public class TarFiles implements Closeable {
}
public void flush() throws IOException {
+ checkInitialised();
lock.readLock().lock();
try {
writer.flush();
@@ -592,6 +625,7 @@ public class TarFiles implements Closeable {
}
public void writeSegment(UUID id, byte[] buffer, int offset, int length,
GCGeneration generation, Set<UUID> references, Set<String> binaryReferences)
throws IOException {
+ checkInitialised();
lock.writeLock().lock();
try {
long size = writer.writeEntry(
@@ -643,6 +677,7 @@ public class TarFiles implements Closeable {
}
void newWriter() throws IOException {
+ checkInitialised();
lock.writeLock().lock();
try {
internalNewWriter();
@@ -652,6 +687,7 @@ public class TarFiles implements Closeable {
}
public CleanupResult cleanup(CleanupContext context) throws IOException {
+ checkInitialised();
CleanupResult result = new CleanupResult();
result.removableFiles = new ArrayList<>();
result.reclaimedSegmentIds = new HashSet<>();
@@ -803,6 +839,7 @@ public class TarFiles implements Closeable {
}
public void collectBlobReferences(Consumer<String> collector,
Predicate<GCGeneration> reclaim) throws IOException {
+ checkInitialised();
Node head;
lock.writeLock().lock();
try {
@@ -889,6 +926,7 @@ public class TarFiles implements Closeable {
}
public FileReaper createFileReaper() {
+ checkInitialised();
return new FileReaper(archiveManager);
}
}
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index 47f080370d..3a812f83f8 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++
b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -30,6 +30,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -44,6 +45,8 @@ import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Predicate;
import org.apache.jackrabbit.oak.commons.Buffer;
+import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.SegmentId;
import
org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndex;
import
org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndexLoader;
import
org.apache.jackrabbit.oak.segment.file.tar.binaries.InvalidBinaryReferencesIndexException;
@@ -178,10 +181,11 @@ public class TarReader implements Closeable {
log.info("Regenerating tar file {}", file);
try (TarWriter writer = new TarWriter(archiveManager, file)) {
+ Map<SegmentId, Segment> segmentMap = new HashMap<>(entries.size());
+
for (Entry<UUID, byte[]> entry : entries.entrySet()) {
try {
recovery.recoverEntry(entry.getKey(), entry.getValue(),
new EntryRecovery() {
-
@Override
public void recoverEntry(long msb, long lsb, byte[]
data, int offset, int size, GCGeneration generation) throws IOException {
writer.writeEntry(msb, lsb, data, offset, size,
generation);
@@ -197,6 +201,18 @@ public class TarReader implements Closeable {
writer.addBinaryReference(generation, segmentId,
reference);
}
+ public Segment getSegment(SegmentId id) {
+ return segmentMap.get(id);
+ }
+
+ public void addSegment(Segment segment) {
+ segmentMap.put(segment.getSegmentId(), segment);
+ }
+
+ @Override
+ public Map<SegmentId, Segment> getRecoveredSegments() {
+ return segmentMap;
+ }
});
} catch (IOException e) {
throw new IOException(String.format("Unable to recover
entry %s for file %s", entry.getKey(), file), e);
diff --git
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
new file mode 100644
index 0000000000..032b24d0de
--- /dev/null
+++
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class IdMappingBlobStore implements BlobStore {
+
+ private final MemoryBlobStore bs = new MemoryBlobStore();
+
+ private final Map<String, String> ids = new HashMap<>();
+
+ @Override
+ public String writeBlob(InputStream inputStream) throws IOException {
+ String in = bs.writeBlob(inputStream);
+ String out = generateId();
+ ids.put(out, in);
+ return out;
+ }
+
+ @Override
+ public String writeBlob(InputStream inputStream, BlobOptions options)
throws IOException {
+ return writeBlob(inputStream);
+ }
+
+ @Override
+ public int readBlob(String s, long l, byte[] bytes, int i, int i1) throws
IOException {
+ return bs.readBlob(mapId(s), l, bytes, i, i1);
+ }
+
+ @Override
+ public long getBlobLength(String s) throws IOException {
+ return bs.getBlobLength(mapId(s));
+ }
+
+ @Override
+ public InputStream getInputStream(String s) throws IOException {
+ return bs.getInputStream(mapId(s));
+ }
+
+ @Override
+ public String getBlobId(@NotNull String s) {
+ return bs.getBlobId(s);
+ }
+
+ @Override
+ public String getReference(@NotNull String s) {
+ return bs.getBlobId(mapId(s));
+ }
+
+ @Override
+ public void close() throws Exception {
+ bs.close();
+ }
+
+ private String mapId(String in) {
+ String out = ids.get(in);
+ if (out == null) {
+ throw new IllegalArgumentException("in");
+ }
+ return out;
+ }
+
+ protected abstract String generateId();
+}
diff --git
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
similarity index 67%
copy from
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
copy to
oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
index 34bef73a9c..2b51647bc7 100644
---
a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
+++
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
@@ -15,17 +15,17 @@
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.segment.file.tar;
+package org.apache.jackrabbit.oak.segment;
-import java.io.IOException;
-import java.util.UUID;
+import org.apache.jackrabbit.guava.common.base.Strings;
-public interface EntryRecovery {
+public class LongIdMappingBlobStore extends IdMappingBlobStore {
- void recoverEntry(long msb, long lsb, byte[] data, int offset, int size,
GCGeneration generation) throws IOException;
+ private static int next;
- void recoverGraphEdge(UUID from, UUID to);
-
- void recoverBinaryReference(GCGeneration generation, UUID segmentId,
String reference);
+ @Override
+ protected String generateId() {
+ return Strings.repeat("0", Segment.BLOB_ID_SMALL_LIMIT) + next++;
+ }
}
diff --git
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
index 9043799682..d2e6f0d355 100644
---
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
+++
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
@@ -19,20 +19,15 @@
package org.apache.jackrabbit.oak.segment.file;
-import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.segment.LongIdMappingBlobStore;
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.file.tar.SegmentTarManager;
import org.apache.jackrabbit.oak.segment.file.tar.SegmentTarWriter;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
@@ -41,20 +36,34 @@ import
org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitor;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.jetbrains.annotations.NotNull;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static
org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static org.junit.Assert.*;
+
public class FileStoreTest {
private static final String FAILED_TO_WRITE_ON_CLOSE = "Failed to write
to the archive on closing";
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
- private File getFileStoreFolder() {
- return folder.getRoot();
+ private File getFileStoreFolder() throws IOException {
+ return folder.newFolder("segmentstore");
}
@Test
@@ -140,6 +149,73 @@ public class FileStoreTest {
assertEquals("already shut down", closeEx.getMessage());
}
+ @Test
+ public void testRecovery_FileStore_withExternalBlobStore() throws
InvalidFileStoreVersionException, IOException, CommitFailedException {
+
+ File segmentStore = getFileStoreFolder();
+ BlobStore blobStore = new LongIdMappingBlobStore();
+ FileStore fileStore = createFileStore(segmentStore, blobStore);
+
+ SegmentNodeStore segmentNodeStore =
SegmentNodeStoreBuilders.builder(fileStore).build();
+
+ NodeBuilder builder = segmentNodeStore.getRoot().builder();
+ builder.setProperty("foo", "bar");
+
+ Blob blob = builder.createBlob(new ZeroStream(100));
+
+
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.flush();
+
+ // create another segment
+
+ builder.setProperty("binaryProperty", blob);
+
+ builder.setProperty("foo1", "bar1");
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.flush();
+
+ // copy all files form segmentStore to segmentStoreClone
+ // with this, tha last tar archive will not have index, graph and
binary references
+ File segmentStoreClone = folder.newFolder("segmentstore-clone");
+ Files.walk(segmentStore.toPath())
+ .forEach(source -> {
+ try {
+ Path target =
segmentStoreClone.toPath().resolve(segmentStore.toPath().relativize(source));
+ Files.copy(source, target,
StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ });
+
+
+ fileStore.close();
+
+ // Start new FileStore, which should be able to recover successfully
despite now having the index, graph and binary references
+ try {
+ fileStore = createFileStore(segmentStoreClone, blobStore);
+ } catch (Exception e){
+ fail("Should not throw exception" + e);
+ }
+
+ segmentNodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+
+
+ builder = segmentNodeStore.getRoot().builder();
+
+ assertNotNull(builder.getProperty("foo"));
+ assertNotNull(builder.getProperty("foo1"));
+ blob = builder.getProperty("binaryProperty").getValue(Type.BINARY);
+ assertNotNull(blob);
+ }
+
+ public FileStore createFileStore(File segmentStore, BlobStore blobStore)
throws IOException, InvalidFileStoreVersionException {
+ return fileStoreBuilder(segmentStore)
+ .withBinariesInlineThreshold(0)
+ .withBlobStore(blobStore)
+ .build();
+ }
+
private static int counter = 0;
private static void writeData(FileStore fileStore, int size) throws
IOException {
diff --git
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
index 39deea6ea2..37693d6ede 100644
---
a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
+++
b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
@@ -24,11 +24,7 @@ import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static java.util.UUID.randomUUID;
import static
org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
@@ -40,6 +36,7 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
+import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.file.tar.TarFiles.CleanupResult;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
@@ -397,4 +394,52 @@ public class TarFilesTest {
assertTrue(result.getReclaimedSegmentIds().isEmpty());
assertEquals(0, result.getReclaimedSize());
}
+
+ @Test
+ public void testUninitialisedTarFiles() throws IOException {
+ tarFiles = TarFiles.builder()
+ .withDirectory(folder.getRoot())
+ .withTarRecovery((id, data, recovery) -> {
+ // Intentionally left blank
+ })
+ .withIOMonitor(new IOMonitorAdapter())
+ .withFileStoreMonitor(new FileStoreMonitorAdapter())
+ .withMaxFileSize(MAX_FILE_SIZE)
+ .withRemoteStoreMonitor(new RemoteStoreMonitorAdapter())
+ .withInitialisedReadersAndWriters(false)
+ .build();
+
+ assertEquals(0, tarFiles.readerCount());
+ assertEquals(0, tarFiles.segmentCount());
+ assertEquals(0, tarFiles.size());
+ assertFalse(tarFiles.containsSegment(0l, 0l));
+ assertNull(tarFiles.readSegment(0l, 0l));
+
+ tarFiles.toString();
+ assertThrows(IllegalRepositoryStateException.class, () ->
tarFiles.flush());
+ assertThrows(IllegalRepositoryStateException.class, () ->
writeSegment(randomUUID()));
+ assertThrows(IllegalRepositoryStateException.class, () ->
tarFiles.cleanup(new CleanupContext() {
+ @Override
+ public Collection<UUID> initialReferences() {
+ return emptySet();
+ }
+
+ @Override
+ public boolean shouldReclaim(UUID id, GCGeneration generation,
boolean referenced) {
+ return false;
+ }
+
+ @Override
+ public boolean shouldFollow(UUID from, UUID to) {
+ return false;
+ }
+ }));
+
+ assertThrows(IllegalRepositoryStateException.class, () ->
tarFiles.collectBlobReferences(r -> {}, g -> false));
+ assertFalse(tarFiles.getSegmentIds().iterator().hasNext());
+ assertTrue(tarFiles.getIndices().isEmpty());
+ assertTrue(tarFiles.getGraph("").isEmpty());
+
+ assertThrows(IllegalRepositoryStateException.class, () ->
tarFiles.createFileReaper());
+ }
}
