(jackrabbit-oak) branch OAK-10836 deleted (was ff7f1001de)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10836 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git was ff7f1001de OAK-10836: Improve message 'Another copy of the index update is already running' The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(jackrabbit-oak) branch trunk updated: OAK-10836: Improve message 'Another copy of the index update is already running' (#1495)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 14378839c9 OAK-10836: Improve message 'Another copy of the index
update is already running' (#1495)
14378839c9 is described below
commit 14378839c9c149f5b6747429e7fd360f4659f86b
Author: Amit Jain
AuthorDate: Wed Aug 28 12:45:12 2024 +0530
OAK-10836: Improve message 'Another copy of the index update is already
running' (#1495)
Minor: Changed message
---
.../java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
index 14116cf766..e840a98957 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
@@ -107,7 +107,7 @@ public class AsyncIndexUpdate implements Runnable,
Closeable {
* Name of service property which determines the name of Async task
*/
public static final String PROP_ASYNC_NAME = "oak.async";
-private static final String CONCURRENT_EXCEPTION_MSG ="Another copy of the
index update is already running; skipping this update. ";
+private static final String CONCURRENT_EXCEPTION_MSG ="The index was not
updated. Waiting for the lease to expire (another copy might be still running);
skipping this update. ";
private static final Logger log = LoggerFactory
.getLogger(AsyncIndexUpdate.class);
(jackrabbit-oak) branch trunk updated: Issues/oak 10781 (#1518)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 868e308a1d Issues/oak 10781 (#1518)
868e308a1d is described below
commit 868e308a1d7832f086d397d528b2b786bd6e46cf
Author: Tushar <145645280+t-r...@users.noreply.github.com>
AuthorDate: Tue Jun 18 14:45:58 2024 +0530
Issues/oak 10781 (#1518)
OAK-10781: Access Token refresh in oak-blob-cloud-azure
* only one access token will exist per class instance, and there will only
be one token refresh exectuor that will check and refresh the access token.
* close executor
* reduce token refresh delay to 1 minute
---
oak-blob-cloud-azure/pom.xml | 7 +++
.../blobstorage/AzureBlobContainerProvider.java| 73 +-
.../azure/blobstorage/AzureBlobStoreBackend.java | 1 +
.../azure/blobstorage/AzureDataStoreUtils.java | 10 +--
.../blob/cloud/azure/blobstorage/TestAzureDS.java | 10 +++
5 files changed, 66 insertions(+), 35 deletions(-)
diff --git a/oak-blob-cloud-azure/pom.xml b/oak-blob-cloud-azure/pom.xml
index c058e10419..328091dfce 100644
--- a/oak-blob-cloud-azure/pom.xml
+++ b/oak-blob-cloud-azure/pom.xml
@@ -351,6 +351,13 @@
testcontainers
test
+
+org.apache.jackrabbit
+oak-commons
+${project.version}
+tests
+test
+
diff --git
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
index 2f6d6b4f47..01522248b0 100644
---
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
+++
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java
@@ -42,7 +42,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
-import java.io.IOException;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import java.time.Instant;
@@ -51,6 +50,7 @@ import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.EnumSet;
+import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Executors;
@@ -70,9 +70,12 @@ public class AzureBlobContainerProvider implements Closeable
{
private final String tenantId;
private final String clientId;
private final String clientSecret;
+private ClientSecretCredential clientSecretCredential;
+private AccessToken accessToken;
+private StorageCredentialsToken storageCredentialsToken;
private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L;
private static final long TOKEN_REFRESHER_DELAY = 1L;
-private static final ScheduledExecutorService executorService =
Executors.newSingleThreadScheduledExecutor();
+private final ScheduledExecutorService executorService =
Executors.newSingleThreadScheduledExecutor();
private AzureBlobContainerProvider(Builder builder) {
this.azureConnectionString = builder.azureConnectionString;
@@ -177,13 +180,17 @@ public class AzureBlobContainerProvider implements
Closeable {
public CloudBlobContainer getBlobContainer(@Nullable BlobRequestOptions
blobRequestOptions) throws DataStoreException {
// connection string will be given preference over service principals
/ sas / account key
if (StringUtils.isNotBlank(azureConnectionString)) {
+log.debug("connecting to azure blob storage via
azureConnectionString");
return Utils.getBlobContainer(azureConnectionString,
containerName, blobRequestOptions);
} else if (authenticateViaServicePrincipal()) {
+log.debug("connecting to azure blob storage via service principal
credentials");
return getBlobContainerFromServicePrincipals(blobRequestOptions);
} else if (StringUtils.isNotBlank(sasToken)) {
+log.debug("connecting to azure blob storage via sas token");
final String connectionStringWithSasToken =
Utils.getConnectionStringForSas(sasToken, blobEndpoint, accountName);
return Utils.getBlobContainer(connectionStringWithSasToken,
containerName, blobRequestOptions);
}
+log.debug("connecting to azure blob storage via access key");
final String connectionStringWithAccountKey =
Utils.getConnectionString(accountName, accountKey, blobEndpoint);
return Utils.getBlobContainer(connectionStringWithAccountKey,
containerName,
(jackrabbit-oak) 01/01: OAK-10836: Improve message 'Another copy of the index update is already running'
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10836
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit ff7f1001de9f5ebde682436b57ae2159abf7fd87
Author: amjain
AuthorDate: Thu May 30 12:58:21 2024 +0530
OAK-10836: Improve message 'Another copy of the index update is already
running'
Minor: Changed message
---
.../java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
index 83fb5282ac..37030734aa 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
@@ -107,7 +107,7 @@ public class AsyncIndexUpdate implements Runnable,
Closeable {
* Name of service property which determines the name of Async task
*/
public static final String PROP_ASYNC_NAME = "oak.async";
-private static final String CONCURRENT_EXCEPTION_MSG ="Another copy of the
index update is already running; skipping this update. ";
+private static final String CONCURRENT_EXCEPTION_MSG ="The index was not
updated. Waiting for the lease to expire (another copy might be still running);
skipping this update. ";
private static final Logger log = LoggerFactory
.getLogger(AsyncIndexUpdate.class);
(jackrabbit-oak) branch OAK-10836 created (now ff7f1001de)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10836 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at ff7f1001de OAK-10836: Improve message 'Another copy of the index update is already running' This branch includes the following new commits: new ff7f1001de OAK-10836: Improve message 'Another copy of the index update is already running' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] branch OAK-10495 deleted (was 96f64e1d42)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10495 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git was 96f64e1d42 OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[jackrabbit-oak] branch trunk updated: OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests (#1157)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 13889a17d1 OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex*
tests (#1157)
13889a17d1 is described below
commit 13889a17d143d7a24b2907f6626f3c03f7ed07a1
Author: Amit Jain
AuthorDate: Tue Oct 17 14:45:10 2023 +0530
OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests (#1157)
- Ignored tests
Co-authored-by: amjain
---
.../java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java | 4
1 file changed, 4 insertions(+)
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
index cb41b1093d..2b632d64ba 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
@@ -60,6 +60,7 @@ import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
@@ -168,6 +169,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
}
@Test
+@Ignore("OAK-10495")
public void parallelReindex() throws Exception {
LOG.info("Starting parallelReindex");
System.setProperty(IndexStoreUtils.OAK_INDEXER_USE_LZ4, "false");
@@ -176,6 +178,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
}
@Test
+@Ignore("OAK-10495")
public void parallelReindexWithLZ4() throws Exception {
LOG.info("Starting parallelReindexWithLZ4");
System.setProperty(OAK_INDEXER_USE_LZ4, "true");
@@ -378,6 +381,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
}
@Test
+@Ignore("OAK-10495")
public void testParallelIndexing() throws Exception {
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_PARALLEL_INDEX,
"true");
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_THREAD_POOL_SIZE, "2");
[jackrabbit-oak] branch OAK-10495 updated: OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10495
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/OAK-10495 by this push:
new 96f64e1d42 OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex*
tests
96f64e1d42 is described below
commit 96f64e1d422779ea6ef33dd81bf648f166fbe2a9
Author: amjain
AuthorDate: Tue Oct 17 13:28:02 2023 +0530
OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests
- Ignored testParallelIndexing test as well
---
.../java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
index 69edd00839..f22ecc70fd 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
@@ -380,6 +380,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
}
@Test
+@Ignore("OAK-10495")
public void testParallelIndexing() throws Exception {
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_PARALLEL_INDEX,
"true");
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_THREAD_POOL_SIZE, "2");
[jackrabbit-oak] branch OAK-10495 created (now 6fdbcc3068)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10495 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at 6fdbcc3068 OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests This branch includes the following new commits: new 6fdbcc3068 OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] 01/01: OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10495
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 6fdbcc3068d59dac88d411607ab940b563dfc7d1
Author: amjain
AuthorDate: Tue Oct 17 13:17:23 2023 +0530
OAK-10495: Ignore DocumentStoreIndexerIT#parallelReindex* tests
- Ignored tests
---
.../java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java | 3 +++
1 file changed, 3 insertions(+)
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
index d0dd56bc44..69edd00839 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
@@ -59,6 +59,7 @@ import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
@@ -167,6 +168,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
}
@Test
+@Ignore("OAK-10495")
public void parallelReindex() throws Exception {
LOG.info("Starting parallelReindex");
System.setProperty(IndexStoreUtils.OAK_INDEXER_USE_LZ4, "false");
@@ -175,6 +177,7 @@ public class DocumentStoreIndexerIT extends
LuceneAbstractIndexCommandTest {
}
@Test
+@Ignore("OAK-10495")
public void parallelReindexWithLZ4() throws Exception {
LOG.info("Starting parallelReindexWithLZ4");
System.setProperty(OAK_INDEXER_USE_LZ4, "true");
[jackrabbit-oak] branch trunk updated: OAK-10384: Fix stripping of large indexed ordered properties (#1071)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1e55c01b22 OAK-10384: Fix stripping of large indexed ordered
properties (#1071)
1e55c01b22 is described below
commit 1e55c01b22396239653549b3684bd9d71c606307
Author: Amit Jain
AuthorDate: Mon Sep 11 10:17:33 2023 +0530
OAK-10384: Fix stripping of large indexed ordered properties (#1071)
- Truncate BytesRef value and handle surrogates correctly (Code from Thomas
Mueller)
---
.../plugins/index/lucene/LuceneDocumentMaker.java | 62 ++-
.../lucene/LuceneLargeStringPropertyTest.java | 116 +++--
2 files changed, 163 insertions(+), 15 deletions(-)
diff --git
a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneDocumentMaker.java
b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneDocumentMaker.java
index d41461c62e..06167fb03b 100644
---
a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneDocumentMaker.java
+++
b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneDocumentMaker.java
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.plugins.index.lucene;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -287,11 +288,10 @@ public class LuceneDocumentMaker extends
FulltextDocumentMaker {
new
BytesRef(property.getValue(Type.BOOLEAN).toString()));
} else if (tag == Type.STRING.tag()) {
String stringValue = property.getValue(Type.STRING);
-if (stringValue.length() > STRING_PROPERTY_MAX_LENGTH){
-log.warn("Truncating property {} having length {} at
path:[{}] as it is > {}", name, stringValue.length(), this.path,
STRING_PROPERTY_MAX_LENGTH);
-stringValue = stringValue.substring(0,
STRING_PROPERTY_MAX_LENGTH);
-}
-f = new SortedDocValuesField(name, new BytesRef(stringValue));
+// Truncate the value as lucene limits the length of a
SortedDocValueField string to
+// STRING_PROPERTY_MAX_LENGTH(32766 bytes) and throws
exception if over the limit
+f = new SortedDocValuesField(name, getTruncatedBytesRef(name,
stringValue, this.path,
+STRING_PROPERTY_MAX_LENGTH));
}
if (f != null && includePropertyValue(property, 0, pd)) {
@@ -316,6 +316,58 @@ public class LuceneDocumentMaker extends
FulltextDocumentMaker {
return fieldAdded;
}
+/**
+ * Returns a {@code BytesRef} object constructed from the given {@code
String} value and also truncates the length
+ * of the {@code BytesRef} object to the specified {@code maxLength},
ensuring that the multi-byte sequences are
+ * properly truncated.
+ *
+ * The {@code BytesRef} object is created from the provided {@code
String} value using UTF-8 encoding. As a result, its length
+ * can exceed that of the {@code String} value, since Java strings use
UTF-16 encoding. This necessitates appropriate truncation.
+ *
+ * Multi-byte sequences will be of the form {@code 11xx 10xx
10xx 10xx}.
+ * The method first truncates continuation bytes, which start with {@code
10} in binary. It then truncates the head byte, which
+ * starts with {@code 11}. Both truncation operations use a binary mask of
{@code 1100}.
+ *
+ * @param prop the name of the property
+ * @param value the string property value to convert into a {@code
BytesRef} object
+ * @param path the path of the node
+ * @param maxLength the maximum length for the {@code BytesRef} object
+ * @return the truncated {@code BytesRef} object
+ */
+protected static BytesRef getTruncatedBytesRef(String prop, String value,
String path, int maxLength) {
+BytesRef ref = new BytesRef(value);
+if (ref.length <= maxLength) {
+return ref;
+}
+
+log.trace("Property {} at path:[{}] has value {}", prop, path, value);
+log.info("Truncating property {} at path:[{}] as length after encoding
{} is > {} ",
+prop, path, ref.length, maxLength);
+
+int end = maxLength - 1;
+// skip over tails of utf-8 multi-byte sequences (up to 3 bytes)
+while ((ref.bytes[end] & 0b1100) == 0b1000) {
+end--;
+}
+// remove one head of a utf-8 multi-byte sequence (at most 1)
+if ((ref.bytes[end] & 0b1100) == 0b1100) {
+end--;
+}
+byte[] truncated
[jackrabbit-oak] branch OAK-10253 deleted (was 85f8024de3)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10253 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git was 85f8024de3 OAK-10253: Option to only collect references when calling checkConsistency The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[jackrabbit-oak] branch trunk updated: OAK-10253: Option to only collect references when calling checkConsis… (#947)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new dff1d17362 OAK-10253: Option to only collect references when calling
checkConsis… (#947)
dff1d17362 is described below
commit dff1d173625d9ffe03a39efe6004b42214699a95
Author: Amit Jain
AuthorDate: Tue May 23 10:53:32 2023 +0530
OAK-10253: Option to only collect references when calling checkConsis…
(#947)
- Add markOnly option for the check-consistency command
- document option for oak-run
---
.../oak/plugins/blob/BlobGarbageCollector.java | 9 ++
.../plugins/blob/MarkSweepGarbageCollector.java| 101 +++--
.../jackrabbit/oak/plugins/blob/BlobGCTest.java| 16
oak-run/README.md | 6 +-
.../jackrabbit/oak/run/DataStoreCommand.java | 2 +-
.../jackrabbit/oak/run/DataStoreOptions.java | 10 +-
.../jackrabbit/oak/run/DataStoreCommandTest.java | 55 +++
7 files changed, 128 insertions(+), 71 deletions(-)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
index 1e7598f15f..f5164f6e7a 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
@@ -59,6 +59,15 @@ public interface BlobGarbageCollector {
*/
long checkConsistency() throws Exception;
+/**
+ * Collects the blob references and consolidates references from other
repositories if available in the DataStore.
+ * Adds relevant metrics.
+ *
+ * @return
+ * @throws Exception
+ */
+long checkConsistency(boolean markOnly) throws Exception;
+
/**
* Returns operation statistics
*
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index ccd6eebb05..078f058801 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -685,15 +685,9 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
closeQuietly(writer);
}
}
-
-/**
- * Checks for the DataStore consistency and reports the number of missing
blobs still referenced.
- *
- * @return the missing blobs
- * @throws Exception
- */
+
@Override
-public long checkConsistency() throws Exception {
+public long checkConsistency(boolean markOnly) throws Exception {
consistencyStatsCollector.start();
Stopwatch sw = Stopwatch.createStarted();
@@ -702,47 +696,36 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
long candidates = 0;
try {
-LOG.info("Starting blob consistency check");
-
-// Find all blobs available in the blob store
-ListenableFutureTask blobIdRetriever =
ListenableFutureTask.create(new BlobIdRetriever(fs,
-true));
-executor.execute(blobIdRetriever);
+LOG.info("Starting blob consistency check with markOnly = {}",
markOnly);
// Mark all used blob references
+// Create a time marker in the data store if applicable
+String uniqueSuffix = UUID.randomUUID().toString();
+
GarbageCollectionType.get(blobStore).addMarkedStartMarker(blobStore, repoId,
uniqueSuffix);
iterateNodeTree(fs, true);
// Move the marked references file to the data store meta area if
applicable
-String uniqueSuffix = UUID.randomUUID().toString();
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs,
repoId, uniqueSuffix);
consistencyStatsCollector.updateMarkDuration(sw.elapsed(TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS);
-try {
-blobIdRetriever.get();
-} catch (ExecutionException e) {
-LOG.warn("Error occurred while fetching all the blobIds from
the BlobStore");
-threw = false;
-throw e;
-}
-
if (SharedDataStoreUtils.isShared(blobStore)) {
// Retrieve all other marked present in the datastore
List refFiles =
-((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
[jackrabbit-oak] branch OAK-10253 updated (408c459d31 -> 85f8024de3)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10253 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git from 408c459d31 OAK-10253: Option to only collect references when calling checkConsistency add 85f8024de3 OAK-10253: Option to only collect references when calling checkConsistency No new revisions were added by this update. Summary of changes: oak-run/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
[jackrabbit-oak] 01/01: OAK-10253: Option to only collect references when calling checkConsistency
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10253
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 408c459d312b036549112bd8dd21d404f6e59e2d
Author: amjain
AuthorDate: Mon May 22 12:30:50 2023 +0530
OAK-10253: Option to only collect references when calling checkConsistency
- Add markOnly option for the check-consistency command
---
.../oak/plugins/blob/BlobGarbageCollector.java | 9 ++
.../plugins/blob/MarkSweepGarbageCollector.java| 101 +++--
.../jackrabbit/oak/plugins/blob/BlobGCTest.java| 16
.../jackrabbit/oak/run/DataStoreCommand.java | 2 +-
.../jackrabbit/oak/run/DataStoreOptions.java | 10 +-
.../jackrabbit/oak/run/DataStoreCommandTest.java | 55 +++
6 files changed, 125 insertions(+), 68 deletions(-)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
index 1e7598f15f..f5164f6e7a 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
@@ -59,6 +59,15 @@ public interface BlobGarbageCollector {
*/
long checkConsistency() throws Exception;
+/**
+ * Collects the blob references and consolidates references from other
repositories if available in the DataStore.
+ * Adds relevant metrics.
+ *
+ * @return
+ * @throws Exception
+ */
+long checkConsistency(boolean markOnly) throws Exception;
+
/**
* Returns operation statistics
*
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index ccd6eebb05..078f058801 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -685,15 +685,9 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
closeQuietly(writer);
}
}
-
-/**
- * Checks for the DataStore consistency and reports the number of missing
blobs still referenced.
- *
- * @return the missing blobs
- * @throws Exception
- */
+
@Override
-public long checkConsistency() throws Exception {
+public long checkConsistency(boolean markOnly) throws Exception {
consistencyStatsCollector.start();
Stopwatch sw = Stopwatch.createStarted();
@@ -702,47 +696,36 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
long candidates = 0;
try {
-LOG.info("Starting blob consistency check");
-
-// Find all blobs available in the blob store
-ListenableFutureTask blobIdRetriever =
ListenableFutureTask.create(new BlobIdRetriever(fs,
-true));
-executor.execute(blobIdRetriever);
+LOG.info("Starting blob consistency check with markOnly = {}",
markOnly);
// Mark all used blob references
+// Create a time marker in the data store if applicable
+String uniqueSuffix = UUID.randomUUID().toString();
+
GarbageCollectionType.get(blobStore).addMarkedStartMarker(blobStore, repoId,
uniqueSuffix);
iterateNodeTree(fs, true);
// Move the marked references file to the data store meta area if
applicable
-String uniqueSuffix = UUID.randomUUID().toString();
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs,
repoId, uniqueSuffix);
consistencyStatsCollector.updateMarkDuration(sw.elapsed(TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS);
-try {
-blobIdRetriever.get();
-} catch (ExecutionException e) {
-LOG.warn("Error occurred while fetching all the blobIds from
the BlobStore");
-threw = false;
-throw e;
-}
-
if (SharedDataStoreUtils.isShared(blobStore)) {
// Retrieve all other marked present in the datastore
List refFiles =
-((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
// Get all the repositories registered
List repoFiles =
-((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStor
[jackrabbit-oak] branch OAK-10253 created (now 408c459d31)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10253 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at 408c459d31 OAK-10253: Option to only collect references when calling checkConsistency This branch includes the following new commits: new 408c459d31 OAK-10253: Option to only collect references when calling checkConsistency The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] branch OAK-10249 deleted (was db94c4dad5)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10249 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git was db94c4dad5 OAK-10249: Reduce logging in the datastore check and gc process The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[jackrabbit-oak] branch trunk updated: OAK-10249: Reduce logging in the datastore check and gc process (#945)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9da4894b8c OAK-10249: Reduce logging in the datastore check and gc
process (#945)
9da4894b8c is described below
commit 9da4894b8cc75fcb7929c77b4e4e68e9a2c8c80e
Author: Amit Jain
AuthorDate: Fri May 19 19:20:29 2023 +0530
OAK-10249: Reduce logging in the datastore check and gc process (#945)
- only log sizes for individual files to debug
- log blob ids only at 100k
- ignore in memory blobs as anyways not relevant
---
.../apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java | 4 ++--
.../src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index 99987cd99f..ccd6eebb05 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -421,7 +421,7 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
lineIterator.forEachRemaining(line -> {
String id = line.split(DELIM)[0];
long length = DataStoreBlobStore.BlobId.of(id).getLength();
-LOG.info("Blob {} has size {}", id, length);
+LOG.debug("Blob {} has size {}", id, length);
stats.getCollector().updateNumBlobReferences(1);
@@ -662,7 +662,7 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
saveBatchToFile(idBatch, writer);
}
-if (count.get() % getBatchCount() == 0) {
+if (count.get() > 0 && count.get() %
getBatchCount() == 0) {
LOG.info("Collected ({}) blob references",
count.get());
}
} catch (Exception e) {
diff --git
a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
index e0143a1994..fa820815d0 100644
---
a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
+++
b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
@@ -257,7 +257,7 @@ public final class FileIOUtils {
writeAsLine(writer, transformer.apply(iterator.next()),
escape);
count++;
if (logger != null) {
-if (count % 1000 == 0) {
+if (count % 10 == 0) {
logger.info(Strings.nullToEmpty(message) + count);
}
}
[jackrabbit-oak] branch OAK-10249 created (now db94c4dad5)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10249 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at db94c4dad5 OAK-10249: Reduce logging in the datastore check and gc process This branch includes the following new commits: new db94c4dad5 OAK-10249: Reduce logging in the datastore check and gc process The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] 01/01: OAK-10249: Reduce logging in the datastore check and gc process
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10249
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit db94c4dad5d7b0d737dae54d1a1aa3416e0c927f
Author: amjain
AuthorDate: Fri May 19 14:07:49 2023 +0530
OAK-10249: Reduce logging in the datastore check and gc process
- only log sizes for individual files to debug
- log blob ids only at 100k
- ignore in memory blobs as anyways not relevant
---
.../apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java | 4 ++--
.../src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index 99987cd99f..ccd6eebb05 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -421,7 +421,7 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
lineIterator.forEachRemaining(line -> {
String id = line.split(DELIM)[0];
long length = DataStoreBlobStore.BlobId.of(id).getLength();
-LOG.info("Blob {} has size {}", id, length);
+LOG.debug("Blob {} has size {}", id, length);
stats.getCollector().updateNumBlobReferences(1);
@@ -662,7 +662,7 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
saveBatchToFile(idBatch, writer);
}
-if (count.get() % getBatchCount() == 0) {
+if (count.get() > 0 && count.get() %
getBatchCount() == 0) {
LOG.info("Collected ({}) blob references",
count.get());
}
} catch (Exception e) {
diff --git
a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
index e0143a1994..fa820815d0 100644
---
a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
+++
b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
@@ -257,7 +257,7 @@ public final class FileIOUtils {
writeAsLine(writer, transformer.apply(iterator.next()),
escape);
count++;
if (logger != null) {
-if (count % 1000 == 0) {
+if (count % 10 == 0) {
logger.info(Strings.nullToEmpty(message) + count);
}
}
[jackrabbit-oak] branch trunk updated: OAK-10162: Fix Version copier with preserveOnTarget to ignore diverge… (#880)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new d242f633c9 OAK-10162: Fix Version copier with preserveOnTarget to
ignore diverge… (#880)
d242f633c9 is described below
commit d242f633c957cd39d4f70f8781a82fcb76a7dfbd
Author: Amit Jain
AuthorDate: Tue Apr 4 10:12:14 2023 +0530
OAK-10162: Fix Version copier with preserveOnTarget to ignore diverge…
(#880)
When preserveOnTarget is true
- Ignoring diverged version history or if no change detected
- Not changing version properties if these properties are null in the
source which prevents linkage to new successor versions being broken
Do not create new HashSet for empty values
Use getProperty() and then check for null instead of has/getProperty()
Simplify test by using existing code to remove a version
-
Co-authored-by: amjain
Co-authored-by: Marcel Reutegger
---
.../oak/plugins/migration/NodeStateCopier.java | 70 +-
.../plugins/migration/version/VersionCopier.java | 88 ++-
.../migration/version/VersionCopierTest.java | 263 -
.../version/ReadWriteVersionManagerUtil.java | 44
4 files changed, 453 insertions(+), 12 deletions(-)
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
index cebe0e1dd3..4087f8e258 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
@@ -16,8 +16,10 @@
*/
package org.apache.jackrabbit.oak.plugins.migration;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
@@ -32,8 +34,11 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collections;
import java.util.Set;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.ImmutableSet.copyOf;
@@ -145,29 +150,74 @@ public class NodeStateCopier {
*
* @param source The NodeState to copy from.
* @param target The NodeBuilder to copy to.
+ * @param preserveOnTarget boolean to indicate no changes on target except
additions
+ * @param path current path
* @return Whether changes were made or not.
*/
-public static boolean copyProperties(NodeState source, NodeBuilder target)
{
+public static boolean copyProperties(NodeState source, NodeBuilder target,
boolean preserveOnTarget, String path) {
boolean hasChanges = false;
// remove removed properties
-for (final PropertyState property : target.getProperties()) {
-final String name = property.getName();
-if (!source.hasProperty(name)) {
-target.removeProperty(name);
-hasChanges = true;
+if (!preserveOnTarget) {
+for (final PropertyState property : target.getProperties()) {
+final String name = property.getName();
+if (!source.hasProperty(name)) {
+target.removeProperty(name);
+hasChanges = true;
+}
}
}
// add new properties and change changed properties
for (PropertyState property : source.getProperties()) {
if (!property.equals(target.getProperty(property.getName( {
-target.setProperty(property);
-hasChanges = true;
+if (!isVersionPropertyEmpty(source, property,
preserveOnTarget, path)) {
+target.setProperty(property);
+hasChanges = true;
+}
}
}
return hasChanges;
}
+
+private static Set getValues(NodeState nodeState, String prop) {
+PropertyState ps = nodeState.getProperty(prop);
+if (ps != null) {
+Iterable values = ps.getValue(Type.STRINGS);
+return StreamSupport.stream(values.spliterator(), false).filter(s
-> !s.isEmpty()).collect(Collectors.toSet());
+}
+return Collections.emptySet();
+}
+
+private static boolean isVersionPropertyEmpty(NodeState source,
PropertySt
[jackrabbit-oak] branch OAK-10162 updated: OAK-10162: Fix Version copier with preserveOnTarget to ignore diverge… (#884)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10162
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/OAK-10162 by this push:
new c70227cb20 OAK-10162: Fix Version copier with preserveOnTarget to
ignore diverge… (#884)
c70227cb20 is described below
commit c70227cb203c11b01278b5624541b45f153b3f9f
Author: Marcel Reutegger
AuthorDate: Mon Apr 3 10:11:10 2023 +0200
OAK-10162: Fix Version copier with preserveOnTarget to ignore diverge…
(#884)
Do not create new HashSet for empty values
Use getProperty() and then check for null instead of has/getProperty()
Simplify test by using existing code to remove a version
---
.../oak/plugins/migration/NodeStateCopier.java | 9 +++--
.../migration/version/VersionCopierTest.java | 29 --
.../version/ReadWriteVersionManagerUtil.java | 44 ++
3 files changed, 57 insertions(+), 25 deletions(-)
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
index 3e4912fc25..4087f8e258 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
@@ -34,7 +34,7 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashSet;
+import java.util.Collections;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -181,11 +181,12 @@ public class NodeStateCopier {
}
private static Set getValues(NodeState nodeState, String prop) {
-if (nodeState.hasProperty(prop)) {
-Iterable values =
nodeState.getProperty(prop).getValue(Type.STRINGS);
+PropertyState ps = nodeState.getProperty(prop);
+if (ps != null) {
+Iterable values = ps.getValue(Type.STRINGS);
return StreamSupport.stream(values.spliterator(), false).filter(s
-> !s.isEmpty()).collect(Collectors.toSet());
}
-return new HashSet<>();
+return Collections.emptySet();
}
private static boolean isVersionPropertyEmpty(NodeState source,
PropertyState property, boolean preserveOnTarget,
diff --git
a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/migration/version/VersionCopierTest.java
b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/migration/version/VersionCopierTest.java
index 4296654ca9..a9ca170a97 100644
---
a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/migration/version/VersionCopierTest.java
+++
b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/migration/version/VersionCopierTest.java
@@ -18,7 +18,6 @@ package org.apache.jackrabbit.oak.plugins.migration.version;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -42,6 +41,7 @@ import
org.apache.jackrabbit.oak.plugins.migration.DescendantsIterator;
import org.apache.jackrabbit.oak.plugins.migration.NodeStateCopier;
import org.apache.jackrabbit.oak.plugins.version.ReadOnlyVersionManager;
import org.apache.jackrabbit.oak.plugins.version.ReadWriteVersionManager;
+import org.apache.jackrabbit.oak.plugins.version.ReadWriteVersionManagerUtil;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -56,9 +56,11 @@ import static
org.apache.jackrabbit.JcrConstants.JCR_PREDECESSORS;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
import static org.apache.jackrabbit.JcrConstants.JCR_ROOTVERSION;
import static org.apache.jackrabbit.JcrConstants.JCR_SUCCESSORS;
+import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM;
import static org.apache.jackrabbit.JcrConstants.JCR_UUID;
import static org.apache.jackrabbit.JcrConstants.JCR_VERSIONHISTORY;
import static org.apache.jackrabbit.JcrConstants.JCR_VERSIONLABELS;
+import static org.apache.jackrabbit.JcrConstants.JCR_VERSIONSTORAGE;
import static org.apache.jackrabbit.JcrConstants.MIX_REFERENCEABLE;
import static org.apache.jackrabbit.JcrConstants.MIX_VERSIONABLE;
import static org.apache.jackrabbit.JcrConstants.NT_UNSTRUCTURED;
@@ -179,7 +181,7 @@ public class VersionCopierTest {
copyVersionStorage(sourceStore, targetStore);
// Remove 1 version for path /foo0 and create a new one
-removeSpecificVersion(targetStore, "/foo0", "1.0", "1.1");
+removeSpecificVersion(targetStore, "/foo0", "1.0");
createNewVersion("/foo0", targetStore, "
[jackrabbit-oak] branch OAK-10162 updated (2923b40bf2 -> 80d2cf9ebb)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10162 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git from 2923b40bf2 OAK-10162: Fix Version copier with preserveOnTarget to ignore diverged history add 80d2cf9ebb OAK-10162: Fix Version copier with preserveOnTarget to ignore diverged history No new revisions were added by this update. Summary of changes: .../oak/plugins/migration/version/VersionCopier.java| 10 -- .../plugins/migration/version/VersionCopierTest.java| 17 ++--- 2 files changed, 22 insertions(+), 5 deletions(-)
[jackrabbit-oak] 01/01: OAK-10162: Fix Version copier with preserveOnTarget to ignore diverged history
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-10162
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 2923b40bf2716ee9393c6d18a639f1b89f87a6b5
Author: amjain
AuthorDate: Wed Mar 29 15:03:12 2023 +0530
OAK-10162: Fix Version copier with preserveOnTarget to ignore diverged
history
When preserveOnTarget is true
- Ignoring diverged version history or if no change detected
- Not changing version properties if these properties are null in the
source which prevents linkage to new successor versions being broken
---
.../oak/plugins/migration/NodeStateCopier.java | 69 -
.../plugins/migration/version/VersionCopier.java | 82 +-
.../migration/version/VersionCopierTest.java | 277 -
3 files changed, 410 insertions(+), 18 deletions(-)
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
index cebe0e1dd3..3e4912fc25 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
@@ -16,8 +16,10 @@
*/
package org.apache.jackrabbit.oak.plugins.migration;
+import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
import org.apache.jackrabbit.oak.spi.commit.CommitHook;
@@ -32,8 +34,11 @@ import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.HashSet;
import java.util.Set;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.ImmutableSet.copyOf;
@@ -145,29 +150,73 @@ public class NodeStateCopier {
*
* @param source The NodeState to copy from.
* @param target The NodeBuilder to copy to.
+ * @param preserveOnTarget boolean to indicate no changes on target except
additions
+ * @param path current path
* @return Whether changes were made or not.
*/
-public static boolean copyProperties(NodeState source, NodeBuilder target)
{
+public static boolean copyProperties(NodeState source, NodeBuilder target,
boolean preserveOnTarget, String path) {
boolean hasChanges = false;
// remove removed properties
-for (final PropertyState property : target.getProperties()) {
-final String name = property.getName();
-if (!source.hasProperty(name)) {
-target.removeProperty(name);
-hasChanges = true;
+if (!preserveOnTarget) {
+for (final PropertyState property : target.getProperties()) {
+final String name = property.getName();
+if (!source.hasProperty(name)) {
+target.removeProperty(name);
+hasChanges = true;
+}
}
}
// add new properties and change changed properties
for (PropertyState property : source.getProperties()) {
if (!property.equals(target.getProperty(property.getName( {
-target.setProperty(property);
-hasChanges = true;
+if (!isVersionPropertyEmpty(source, property,
preserveOnTarget, path)) {
+target.setProperty(property);
+hasChanges = true;
+}
}
}
return hasChanges;
}
+
+private static Set getValues(NodeState nodeState, String prop) {
+if (nodeState.hasProperty(prop)) {
+Iterable values =
nodeState.getProperty(prop).getValue(Type.STRINGS);
+return StreamSupport.stream(values.spliterator(), false).filter(s
-> !s.isEmpty()).collect(Collectors.toSet());
+}
+return new HashSet<>();
+}
+
+private static boolean isVersionPropertyEmpty(NodeState source,
PropertyState property, boolean preserveOnTarget,
+String path) {
+if (preserveOnTarget) {
+if (property.getName().equals(JcrConstants.JCR_UUID) ||
+property.getName().equals(JcrConstants.JCR_SUCCESSORS) ||
+property.getName().equals(JcrConstants.JCR_PREDECESSORS)) {
+boolean versionPropertyEmpty = getValues(source,
property.getName()).isEmpty();
+if (versionPropertyEmpty) {
+LOG.info("Version Property {} will be skipped for path
{}", prope
[jackrabbit-oak] branch OAK-10162 created (now 2923b40bf2)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-10162 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at 2923b40bf2 OAK-10162: Fix Version copier with preserveOnTarget to ignore diverged history This branch includes the following new commits: new 2923b40bf2 OAK-10162: Fix Version copier with preserveOnTarget to ignore diverged history The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] branch trunk updated: OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong path when FlatFileSplitter used (#826)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new aada6bd4b0 OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore
outputs the wrong path when FlatFileSplitter used (#826)
aada6bd4b0 is described below
commit aada6bd4b0888677b850b5ffd151b079507a0851
Author: Amit Jain
AuthorDate: Mon Jan 23 11:39:13 2023 +0530
OAK-10048: DocumentStoreIndexerBase#buildFlatFileStore outputs the wrong
path when FlatFileSplitter used (#826)
- System property (oak.indexer.sortedFilePath) defines the folder of the
flat file store rather than the current specific file path
- Enable buildFlatFiles to use the folder in the system property defined as
the place to look for files
- In case the folder is split then it uses that to init the split flat file
store
---
.../indexer/document/DocumentStoreIndexerBase.java | 3 +-
.../flatfile/FlatFileNodeStoreBuilder.java | 82 +---
.../document/flatfile/FlatFileSplitter.java| 5 +-
.../indexer/document/flatfile/FlatFileStore.java | 2 +-
.../flatfile/FlatFileNodeStoreBuilderTest.java | 213 +
5 files changed, 238 insertions(+), 67 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
index 90478a2db1..974b6524f8 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
@@ -217,7 +217,8 @@ public abstract class DocumentStoreIndexerBase implements
Closeable{
preferredPathElements.addAll(indexDf.getRelativeNodeNames());
}
Predicate predicate = s ->
indexDefinitions.stream().anyMatch(indexDef ->
indexDef.getPathFilter().filter(s) != PathFilter.Result.EXCLUDE);
-FlatFileStore flatFileStore =
buildFlatFileStoreList(checkpointedState, null, predicate,
preferredPathElements, false, indexDefinitions).get(0);
+FlatFileStore flatFileStore =
buildFlatFileStoreList(checkpointedState, null, predicate,
+preferredPathElements,
IndexerConfiguration.parallelIndexEnabled(), indexDefinitions).get(0);
log.info("FlatFileStore built at {}. To use this flatFileStore in a
reindex step, set System Property-{} with value {}",
flatFileStore.getFlatFileStorePath(),
OAK_INDEXER_SORTED_FILE_PATH, flatFileStore.getFlatFileStorePath());
return flatFileStore;
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
index 8cae86fa1a..177bab3446 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
@@ -21,6 +21,7 @@ package
org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.jackrabbit.oak.commons.Compression;
import org.apache.jackrabbit.oak.index.IndexHelper;
import org.apache.jackrabbit.oak.index.IndexerSupport;
@@ -31,6 +32,7 @@ import
org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.query.NodeStateNodeTypeInfoProvider;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +40,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -65,6 +68,9 @@ public class FlatFileNodeStoreBuilder {
* Allowed values are the values from enum {@link SortStrategyType}
*/
public static final String OAK_INDEXER_SORT_STRATEGY_TYPE =
"oak.indexer.sortStrategyType";
+/**
+ * System property to define the existing folder containing the flat file
store files
+ */
public static final String OAK_INDEXER_SORTED_FILE_PATH =
"oak.indexer.sortedFilePath";
/**
@@ -191,7 +197,8 @@ public class FlatFileNodeStoreBuilder {
logFlags();
comparator = new PathElementComparator(preferre
[jackrabbit-oak] branch trunk updated: OAK-9988: MergeRunnerTest leaves temp files/folders behind
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new ce48d7fec2 OAK-9988: MergeRunnerTest leaves temp files/folders behind
ce48d7fec2 is described below
commit ce48d7fec2e68e456c14a10750c8bbaad649a8cc
Author: Amit Jain
AuthorDate: Tue Dec 13 17:41:45 2022 +0530
OAK-9988: MergeRunnerTest leaves temp files/folders behind
Use TemporaryFolder to create temp directories
---
.../oak/index/indexer/document/flatfile/MergeRunnerTest.java | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
index 7848a295df..a7178a2634 100644
---
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeRunnerTest.java
@@ -23,7 +23,9 @@ import org.apache.jackrabbit.oak.commons.Compression;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.event.Level;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
@@ -49,6 +51,8 @@ public class MergeRunnerTest {
.filter(Level.INFO)
.enable(Level.INFO)
.create();
+@Rule
+public TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
private final String newline = System.lineSeparator();
private final List testFiles = Lists.newArrayList();
private final int threadPoolSize = 1,
@@ -72,7 +76,7 @@ public class MergeRunnerTest {
public void test() throws Exception {
lc.starting();
-File tmpDir = new File(FileUtils.getTempDirectory(),
Long.toString(System.nanoTime())),
+File tmpDir = temporaryFolder.newFolder(),
mergeDir = new File(tmpDir, "merge-reverse"),
sortedFile = new File(tmpDir, "sorted-file.json");
List expectedLogOutput = Lists.newArrayList(),
@@ -160,7 +164,7 @@ public class MergeRunnerTest {
public void testReverse() throws Exception {
lc.starting();
-File tmpDir = new File(FileUtils.getTempDirectory(),
Long.toString(System.nanoTime())),
+File tmpDir = temporaryFolder.newFolder(),
mergeDir = new File(tmpDir, "merge"),
sortedFile = new File(tmpDir, "sorted-file.json");
List expectedLogOutput = Lists.newArrayList(),
[jackrabbit-oak] branch trunk updated: OAK-10013: oak-run-commons: tests fail on Windows Subsystem for Linux (#787)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new e40b0f57c0 OAK-10013: oak-run-commons: tests fail on Windows Subsystem
for Linux (#787)
e40b0f57c0 is described below
commit e40b0f57c0160d70af60849fb676d0943193df15
Author: Amit Jain
AuthorDate: Tue Dec 13 10:06:08 2022 +0530
OAK-10013: oak-run-commons: tests fail on Windows Subsystem for Linux (#787)
-Convert TemporaryFileFolder & adding custom tmpDir from @Before to
@BeforeClass
---
.../indexer/document/flatfile/FlatFileSplitterTest.java | 15 +--
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
index 0824e92bd6..0efc2dd025 100644
---
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
@@ -39,7 +39,8 @@ import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
import org.apache.jackrabbit.oak.spi.commit.EditorHook;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
@@ -79,20 +80,22 @@ public class FlatFileSplitterTest {
private ClassLoader classLoader = getClass().getClassLoader();
private MemoryBlobStore store = new MemoryBlobStore();
private NodeStateEntryReader entryReader = new NodeStateEntryReader(store);
-@Rule
-public TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
+@ClassRule
+public static TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
+@ClassRule
+public static final TestRule restoreClassSystemProperties = new
RestoreSystemProperties();
@Rule
public final TestRule restoreSystemProperties = new
RestoreSystemProperties();
-@Before
-public void setup() throws IOException {
+@BeforeClass
+public static void setup() throws IOException {
try {
System.setProperty("java.io.tmpdir",
temporaryFolder.newFolder("systemp").getAbsolutePath());
} catch (IOException e) {
throw e;
}
}
-
+
@Test
public void ntBaseSkipSplit() throws IOException, IllegalAccessException {
Set splitNodeTypeNames = new HashSet<>();
[jackrabbit-oak] branch trunk updated: OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark phase (#771)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 146ea2e604 OAK-9975: [DSGC] Report cummulative size of referenced
blobs during Mark phase (#771)
146ea2e604 is described below
commit 146ea2e604927215d4cdb9137e60c752bd8c3393
Author: Amit Jain
AuthorDate: Fri Dec 2 11:17:30 2022 +0530
OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark
phase (#771)
- Update size metrics in checkConsistency & mark
---
.../plugins/blob/MarkSweepGarbageCollector.java| 86 +++--
.../oak/plugins/blob/OperationStatsCollector.java | 20
.../oak/plugins/blob/OperationsStatsMBean.java | 12 +++
.../jackrabbit/oak/plugins/blob/BlobGCTest.java| 102 -
4 files changed, 190 insertions(+), 30 deletions(-)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index 085f156c46..d1361e0b52 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -405,13 +405,35 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
// Mark all used references
iterateNodeTree(fs, false);
-
+
+// Get size
+getBlobReferencesSize(fs, stats);
+
// Move the marked references file to the data store meta area if
applicable
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs, repoId,
uniqueSuffix);
LOG.debug("Ending mark phase of the garbage collector");
}
+private static void getBlobReferencesSize(GarbageCollectorFileState fs,
GarbageCollectionOperationStats stats)
+throws IOException {
+try (LineIterator lineIterator = new LineIterator(new
FileReader(fs.getMarkedRefs( {
+lineIterator.forEachRemaining(line -> {
+String id = line.split(DELIM)[0];
+long length = DataStoreBlobStore.BlobId.of(id).getLength();
+LOG.info("Blob {} has size {}", id, length);
+
+stats.getCollector().updateNumBlobReferences(1);
+
+if (length != -1) {
+stats.getCollector().updateBlobReferencesSize(length);
+}
+});
+}
+LOG.info("Blob references found : {} with size : {}",
stats.getNumBlobReferences(),
+stats.getBlobReferencesSize());
+}
+
/**
* Difference phase where the GC candidates are identified.
*
@@ -526,7 +548,7 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
}
if (checkConsistencyAfterGc) {
-
BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs,
consistencyStatsCollector, new File(root));
+
BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs,
consistencyStats);
}
BlobCollectionType.get(blobStore).handleRemoves(blobStore,
fs.getGarbage(), fs.getMarkedRefs());
@@ -689,6 +711,9 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
// Mark all used blob references
iterateNodeTree(fs, true);
+// Move the marked references file to the data store meta area if
applicable
+String uniqueSuffix = UUID.randomUUID().toString();
+GarbageCollectionType.get(blobStore).addMarked(blobStore, fs,
repoId, uniqueSuffix);
consistencyStatsCollector.updateMarkDuration(sw.elapsed(TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS);
try {
@@ -703,6 +728,21 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
// Retrieve all other marked present in the datastore
List refFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
+// Get all the repositories registered
+List repoFiles =
+((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType());
+LOG.info("Repositories registered {}", repoFiles);
+
+// Retrieve repos for which reference files have not been
created
+Set unAvailRepos =
+SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles,
refFiles);
+LOG.info("Repositories with unavailable references {}",
unAvailRepos);
+
+
[jackrabbit-oak] branch OAK-9975 updated (591c2f91e6 -> 7ce6230faa)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-9975 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git from 591c2f91e6 OAK-9975: Throw exception in consistency check if all repositories do not have marked references to push consistent metrics add 7ce6230faa OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark phase No new revisions were added by this update. Summary of changes: .../plugins/blob/MarkSweepGarbageCollector.java| 24 +++--- .../oak/plugins/blob/OperationStatsCollector.java | 4 ++-- .../oak/plugins/blob/OperationsStatsMBean.java | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-)
[jackrabbit-oak] branch trunk updated: OAK-10008: Reduce heap requirement for test DocumentStoreIndexeIT#parallelReindexWithLZ4 (#761)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new cf8c866673 OAK-10008: Reduce heap requirement for test
DocumentStoreIndexeIT#parallelReindexWithLZ4 (#761)
cf8c866673 is described below
commit cf8c866673f07929f286957fa81a9fd9d0608902
Author: Amit Jain
AuthorDate: Mon Nov 21 15:55:34 2022 +0530
OAK-10008: Reduce heap requirement for test
DocumentStoreIndexeIT#parallelReindexWithLZ4 (#761)
- Reduce split size to 2
---
.../java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
index 520dfe2e36..9ede6cd67a 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
@@ -193,7 +193,7 @@ public class DocumentStoreIndexerIT extends
AbstractIndexCommandTest {
System.setProperty("oak.indexer.minMemoryForWork", "1");
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_PARALLEL_INDEX,
"true");
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD,
"0");
-System.setProperty(IndexerConfiguration.PROP_SPLIT_STORE_SIZE, "6");
+System.setProperty(IndexerConfiguration.PROP_SPLIT_STORE_SIZE, "2");
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_THREAD_POOL_SIZE, "2");
System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_USE_ZIP,
"true");
[jackrabbit-oak] branch OAK-9975 updated (5aed51036e -> 591c2f91e6)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-9975 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git from 5aed51036e OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark phase add 76adce992b OAK-9981: Add a option in purgecommand(oak-run) to skip purge operations on active base index (#742) add 4d7ec08f88 OAK-9987:oak-search-elastic depends on vulnerable snakeyaml version (#747) add a89e9773f4 OAK-9990: Update Oak trunk and Oak 1.22 to Jackrabbit 2.20.7 (#748) add 49c2785a1d OAK-9990: Update Oak trunk and Oak 1.22 to Jackrabbit 2.20.7 - fix file mode for pom.xml add 808f02b527 OAK-9978: Test failure: PurgeOldIndexVersionTest. noDeleteIfInvalidIndexOperationVersion (#749) new 2f64a88cc2 Merge branch 'trunk' into OAK-9975 new 591c2f91e6 OAK-9975: Throw exception in consistency check if all repositories do not have marked references to push consistent metrics The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../plugins/blob/MarkSweepGarbageCollector.java| 18 ++ .../jackrabbit/oak/plugins/blob/BlobGCTest.java| 20 ++ oak-jcr/pom.xml| 2 + oak-parent/pom.xml | 2 +- .../oak/indexversion/IndexVersionOperation.java| 71 ++ .../oak/indexversion/PurgeOldIndexVersion.java | 29 +++-- .../oak/run/PurgeOldIndexVersionCommand.java | 6 +- .../oak/indexversion/PurgeOldIndexVersionIT.java | 38 +++- .../oak/indexversion/PurgeOldIndexVersionTest.java | 2 +- oak-search-elastic/pom.xml | 5 -- 10 files changed, 140 insertions(+), 53 deletions(-)
[jackrabbit-oak] 02/02: OAK-9975: Throw exception in consistency check if all repositories do not have marked references to push consistent metrics
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-9975
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 591c2f91e67f81301fd9f5949ab5b4837e71d5d6
Author: amjain
AuthorDate: Fri Nov 18 14:48:32 2022 +0530
OAK-9975: Throw exception in consistency check if all repositories do not
have marked references to push consistent metrics
---
.../oak/plugins/blob/MarkSweepGarbageCollector.java | 18 ++
.../jackrabbit/oak/plugins/blob/BlobGCTest.java | 20
2 files changed, 38 insertions(+)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index 0a172c867e..43b03b52ac 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -711,6 +711,9 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
// Mark all used blob references
iterateNodeTree(fs, true);
+// Move the marked references file to the data store meta area if
applicable
+String uniqueSuffix = UUID.randomUUID().toString();
+GarbageCollectionType.get(blobStore).addMarked(blobStore, fs,
repoId, uniqueSuffix);
consistencyStatsCollector.updateMarkDuration(sw.elapsed(TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS);
try {
@@ -725,6 +728,21 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
// Retrieve all other marked present in the datastore
List refFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
+// Get all the repositories registered
+List repoFiles =
+((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType());
+LOG.info("Repositories registered {}", repoFiles);
+
+// Retrieve repos for which reference files have not been
created
+Set unAvailRepos =
+SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles,
refFiles);
+LOG.info("Repositories with unavailable references {}",
unAvailRepos);
+
+if (!unAvailRepos.isEmpty()) {
+throw new NotAllRepositoryMarkedException("Not all
repositories have marked references available");
+}
+
if (refFiles.size() > 0) {
File temp = new File(root, repoId +
UUID.randomUUID().toString());
copyFile(fs.getMarkedRefs(), temp);
diff --git
a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
index a8cb2c822f..41127c2fe9 100644
---
a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
+++
b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
@@ -63,6 +63,7 @@ import org.apache.jackrabbit.oak.api.blob.BlobAccessProvider;
import org.apache.jackrabbit.oak.api.blob.BlobUpload;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.NotAllRepositoryMarkedException;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -81,6 +82,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.mockito.internal.util.collections.Iterables;
import org.slf4j.Logger;
@@ -94,6 +96,9 @@ public class BlobGCTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+@Rule
+public ExpectedException expectedEx = ExpectedException.none();
protected Whiteboard wb;
@@ -501,6 +506,21 @@ public class BlobGCTest {
assertStatsBean(globalCollector.getConsistencyOperationStats(), 1, 1,
1);
}
+@Test
+public void checkConsistencyFailureNotAllMarked() throws Exception {
+log.info("Staring checkConsistencyFailureNotAllMarked()");
+expectedEx.expect(NotAllRepositoryMarkedException.class);
+
+// Setup a different cluster/repository sha
[jackrabbit-oak] 01/02: Merge branch 'trunk' into OAK-9975
This is an automated email from the ASF dual-hosted git repository. amitj pushed a commit to branch OAK-9975 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git commit 2f64a88cc257c4afe53ca3e8071ce74d54f40c7b Merge: 5aed51036e 808f02b527 Author: amjain AuthorDate: Mon Nov 14 13:12:28 2022 +0530 Merge branch 'trunk' into OAK-9975 oak-jcr/pom.xml| 2 + oak-parent/pom.xml | 2 +- .../oak/indexversion/IndexVersionOperation.java| 71 ++ .../oak/indexversion/PurgeOldIndexVersion.java | 29 +++-- .../oak/run/PurgeOldIndexVersionCommand.java | 6 +- .../oak/indexversion/PurgeOldIndexVersionIT.java | 38 +++- .../oak/indexversion/PurgeOldIndexVersionTest.java | 2 +- oak-search-elastic/pom.xml | 5 -- 8 files changed, 102 insertions(+), 53 deletions(-)
[jackrabbit-oak] branch OAK-9975 created (now 5aed51036e)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-9975 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at 5aed51036e OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark phase This branch includes the following new commits: new 5aed51036e OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark phase The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] 01/01: OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark phase
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-9975
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 5aed51036e87327a9ffb107497c697146de7b7fa
Author: amjain
AuthorDate: Tue Nov 8 16:32:08 2022 +0530
OAK-9975: [DSGC] Report cummulative size of referenced blobs during Mark
phase
- Update size metrics in checkConsistency & mark
---
.../plugins/blob/MarkSweepGarbageCollector.java| 68 --
.../oak/plugins/blob/OperationStatsCollector.java | 20 ++
.../oak/plugins/blob/OperationsStatsMBean.java | 12
.../jackrabbit/oak/plugins/blob/BlobGCTest.java| 82 --
4 files changed, 152 insertions(+), 30 deletions(-)
diff --git
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index 085f156c46..0a172c867e 100644
---
a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++
b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -405,13 +405,35 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
// Mark all used references
iterateNodeTree(fs, false);
-
+
+// Get size
+sizeBlobStoreReferences(fs, stats);
+
// Move the marked references file to the data store meta area if
applicable
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs, repoId,
uniqueSuffix);
LOG.debug("Ending mark phase of the garbage collector");
}
+private static void sizeBlobStoreReferences(GarbageCollectorFileState fs,
GarbageCollectionOperationStats stats)
+throws IOException {
+try (LineIterator lineIterator = new LineIterator(new
FileReader(fs.getMarkedRefs( {
+lineIterator.forEachRemaining(line -> {
+String id = line.split(DELIM)[0];
+long length = DataStoreBlobStore.BlobId.of(id).getLength();
+LOG.info("Blob {} has size {}", id, length);
+
+stats.getCollector().updateNumBlobReferences(1);
+
+if (length != -1) {
+stats.getCollector().updateSizeBlobReferences(length);
+}
+});
+}
+LOG.info("Blob references found : {} with size : {}",
stats.numBlobReferences(),
+stats.sizeBlobReferences());
+}
+
/**
* Difference phase where the GC candidates are identified.
*
@@ -526,7 +548,7 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
}
if (checkConsistencyAfterGc) {
-
BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs,
consistencyStatsCollector, new File(root));
+
BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs,
consistencyStats, new File(root));
}
BlobCollectionType.get(blobStore).handleRemoves(blobStore,
fs.getGarbage(), fs.getMarkedRefs());
@@ -718,6 +740,9 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
}
}
+// Get size
+sizeBlobStoreReferences(fs, consistencyStats);
+
LOG.trace("Starting difference phase of the consistency check");
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getAvailableRefs(),
@@ -1076,8 +1101,8 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
}
void checkConsistencyAfterGC(GarbageCollectableBlobStore blobStore,
GarbageCollectorFileState fs,
-OperationStatsCollector consistencyStatsCollector, File root)
throws IOException {
-consistencyStatsCollector.start();
+GarbageCollectionOperationStats stats, File root) throws
IOException {
+stats.getCollector().start();
Stopwatch sw = Stopwatch.createStarted();
try {
@@ -,12 +1136,15 @@ public class MarkSweepGarbageCollector implements
BlobGarbageCollector {
}
LOG.warn(
"Consistency check failure in the the blob store after
GC : {}", blobStore);
-consistencyStatsCollector.finishFailure();
-consistencyStatsCollector.updateNumDeleted(candidates);
+stats.getCollector().finishFailure();
+stats.getCollector().updateNumDeleted(candidates);
}
+
+// Update the size of the referenced blobs
+sizeBlobStoreReferences(fs, stats);
} finally {
sw.stop();
-
[jackrabbit-oak] branch trunk updated: OAK-9985: DataStoreCopyCommandTest#destinationFromBlobId fails on windows (#746)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new d576b7015e OAK-9985: DataStoreCopyCommandTest#destinationFromBlobId
fails on windows (#746)
d576b7015e is described below
commit d576b7015e61d112a3145a259a05e9761d4bb9fc
Author: Amit Jain
AuthorDate: Tue Nov 8 09:33:30 2022 +0530
OAK-9985: DataStoreCopyCommandTest#destinationFromBlobId fails on windows
(#746)
- Use File.separator to join paths for assertion
---
.../java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java
b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java
index a2fbd76c95..730306254b 100644
---
a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java
+++
b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCopyCommandTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.run;
+import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableSet;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions;
@@ -209,8 +210,8 @@ public class DataStoreCopyCommandTest {
"--out-dir",
outDir.getRoot().getAbsolutePath()
);
-assertEquals(
-outDir.getRoot().getAbsolutePath() +
"/88/97/b9/8897b9025dc534d4a9fa5920569b373cd714c8cfe5d030ca3f5edb25004894a5",
+
assertEquals(Joiner.on(File.separator).join(outDir.getRoot().getAbsolutePath(),
"88",
+"97",
"b9","8897b9025dc534d4a9fa5920569b373cd714c8cfe5d030ca3f5edb25004894a5"),
cmd.getDestinationFromId(BLOB1)
);
}
[jackrabbit-oak] branch trunk updated: OAK-9984: FlatFileSplitterTest reliably failing (#745)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 003aa69836 OAK-9984: FlatFileSplitterTest reliably failing (#745)
003aa69836 is described below
commit 003aa698363c1c35c66a9dd0028744f4c67179fa
Author: Amit Jain
AuthorDate: Tue Nov 8 09:33:07 2022 +0530
OAK-9984: FlatFileSplitterTest reliably failing (#745)
- Replace file length comparisons with file content comparison
- Creating temporary files in the ./target directory
---
.../document/flatfile/FlatFileSplitterTest.java| 66 +-
.../oak/index/DocumentStoreIndexerIT.java | 10
2 files changed, 61 insertions(+), 15 deletions(-)
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
index 0e95be6eb2..0824e92bd6 100644
---
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileSplitterTest.java
@@ -25,6 +25,7 @@ import org.apache.jackrabbit.oak.OakInitializer;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.Compression;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.index.indexer.document.IndexerConfiguration;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
@@ -38,6 +39,7 @@ import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
import org.apache.jackrabbit.oak.spi.commit.EditorHook;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
@@ -49,17 +51,20 @@ import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Scanner;
import java.util.Set;
import java.util.Stack;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
@@ -75,10 +80,19 @@ public class FlatFileSplitterTest {
private MemoryBlobStore store = new MemoryBlobStore();
private NodeStateEntryReader entryReader = new NodeStateEntryReader(store);
@Rule
-public TemporaryFolder temporaryFolder = new TemporaryFolder();
+public TemporaryFolder temporaryFolder = new TemporaryFolder(new
File("target"));
@Rule
public final TestRule restoreSystemProperties = new
RestoreSystemProperties();
-
+
+@Before
+public void setup() throws IOException {
+try {
+System.setProperty("java.io.tmpdir",
temporaryFolder.newFolder("systemp").getAbsolutePath());
+} catch (IOException e) {
+throw e;
+}
+}
+
@Test
public void ntBaseSkipSplit() throws IOException, IllegalAccessException {
Set splitNodeTypeNames = new HashSet<>();
@@ -88,7 +102,8 @@ public class FlatFileSplitterTest {
List flatFileList = splitter.split(false);
assertEquals(1, flatFileList.size());
-assertEquals(flatFile, flatFileList.get(0));
+assertEquals(readAllFilesAsSet(Collections.singletonList(flatFile)),
+readAllFilesAsSet(flatFileList));
}
@Test
@@ -99,7 +114,8 @@ public class FlatFileSplitterTest {
List flatFileList = splitter.split(false);
assertEquals(1, flatFileList.size());
-assertEquals(flatFile, flatFileList.get(0));
+assertEquals(readAllFilesAsSet(Collections.singletonList(flatFile)),
+readAllFilesAsSet(flatFileList));
}
@Test
@@ -111,7 +127,8 @@ public class FlatFileSplitterTest {
List flatFileList = splitter.split(false);
assertEquals(1, flatFileList.size());
-assertEquals(flatFile.length(), flatFileList.get(0).length());
+assertEquals(readAllFilesAsSet(Collections.singletonList(flatFile)),
+readAllFilesAsSet(flatFileList));
}
@Test
@@ -122,10 +139,10 @@ public class FlatFileSplitterTest {
[jackrabbit-oak] branch trunk updated (a440104dd8 -> d7ec57bd07)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git from a440104dd8 Merge pull request #738 from apache/OAK-9977b add d7ec57bd07 OAK-9968: Enable LZ4 compression for parallel indexing (#737) No new revisions were added by this update. Summary of changes: oak-commons/pom.xml| 6 .../oak/commons/sort/ExternalSortTest.java | 24 ++- oak-run-commons/pom.xml| 6 +++- .../flatfile/FlatFileNodeStoreBuilder.java | 6 +++- .../document/flatfile/FlatFileSplitter.java| 4 +++ .../indexer/document/flatfile/LZ4Compression.java | 34 -- .../document/flatfile/FlatFileSplitterTest.java| 6 ++-- .../oak/index/DocumentStoreIndexerIT.java | 8 + 8 files changed, 66 insertions(+), 28 deletions(-) copy oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/binary/BlobByteSource.java => oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/LZ4Compression.java (59%)
[jackrabbit-oak] branch trunk updated: OAK-9807: On DataStore initialization create the reference.key if absent (#634)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3f0338c6a5 OAK-9807: On DataStore initialization create the
reference.key if absent (#634)
3f0338c6a5 is described below
commit 3f0338c6a58076cbce71f089f5c3e9580642e1dd
Author: Amit Jain
AuthorDate: Fri Aug 5 10:12:49 2022 +0530
OAK-9807: On DataStore initialization create the reference.key if absent
(#634)
Create reference.key on init if not present
Co-authored-by: amjain
---
.../azure/blobstorage/AzureBlobStoreBackend.java | 8 +
.../cloud/azure/blobstorage/AzureConstants.java| 5 +++
.../blobstorage/AzureBlobStoreBackendTest.java | 25 ++
.../azure/blobstorage/AzureDataStoreTest.java | 14 +---
.../jackrabbit/oak/blob/cloud/s3/S3Backend.java| 5 ++-
.../oak/blob/cloud/s3/TestS3DataStore.java | 39 +++---
6 files changed, 70 insertions(+), 26 deletions(-)
diff --git
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index e7c0ba5b74..4d94d94105 100644
---
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -220,6 +220,14 @@ public class AzureBlobStoreBackend extends
AbstractSharedBackend {
}
uploadDomainOverride =
properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE,
null);
downloadDomainOverride =
properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE,
null);
+
+// Initialize reference key secret
+boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
+
Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)),
true);
+
+if (createRefSecretOnInit) {
+getOrCreateReferenceKey();
+}
}
catch (StorageException e) {
throw new DataStoreException(e);
diff --git
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
index 0c3956063d..58cf6160c3 100644
---
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
+++
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
@@ -119,5 +119,10 @@ public final class AzureConstants {
*/
public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE =
"presignedHttpUploadURIDomainOverride";
+/**
+ * Property to enable/disable creation of reference secret on init.
+ */
+public static final String AZURE_REF_ON_INIT = "refOnInit";
+
private AzureConstants() { }
}
diff --git
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
index 9cac0cbcaa..d37c6c7237 100644
---
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
@@ -32,6 +32,9 @@ import java.util.EnumSet;
import java.util.Properties;
import java.util.Set;
import java.util.stream.StreamSupport;
+
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
@@ -44,6 +47,8 @@ import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static java.util.stream.Collectors.toSet;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class AzureBlobStoreBackendTest {
@@ -129,6 +134,15 @@ public class AzureBlobStoreBackendTest {
assertReadAccessGranted(azureBlobStoreBackend,
ImmutableSet.of("file"));
}
+@Test
+public void initSecret() throws Exception {
[jackrabbit-oak] 01/01: OAK-9807: On DataStore initialization create the reference.key if absent
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch OAK-9807
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit f640551c26908142d4d4dde35ca77fd7a86fbd16
Author: amjain
AuthorDate: Wed Jul 20 12:13:10 2022 +0530
OAK-9807: On DataStore initialization create the reference.key if absent
Create reference.key on init if not present
---
.../azure/blobstorage/AzureBlobStoreBackend.java | 8 +
.../cloud/azure/blobstorage/AzureConstants.java| 5 +++
.../blobstorage/AzureBlobStoreBackendTest.java | 25 ++
.../azure/blobstorage/AzureDataStoreTest.java | 14 +---
.../jackrabbit/oak/blob/cloud/s3/S3Backend.java| 5 ++-
.../oak/blob/cloud/s3/TestS3DataStore.java | 39 +++---
6 files changed, 70 insertions(+), 26 deletions(-)
diff --git
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
index e7c0ba5b74..4d94d94105 100644
---
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
+++
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
@@ -220,6 +220,14 @@ public class AzureBlobStoreBackend extends
AbstractSharedBackend {
}
uploadDomainOverride =
properties.getProperty(AzureConstants.PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE,
null);
downloadDomainOverride =
properties.getProperty(AzureConstants.PRESIGNED_HTTP_DOWNLOAD_URI_DOMAIN_OVERRIDE,
null);
+
+// Initialize reference key secret
+boolean createRefSecretOnInit = PropertiesUtil.toBoolean(
+
Strings.emptyToNull(properties.getProperty(AzureConstants.AZURE_REF_ON_INIT)),
true);
+
+if (createRefSecretOnInit) {
+getOrCreateReferenceKey();
+}
}
catch (StorageException e) {
throw new DataStoreException(e);
diff --git
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
index 0c3956063d..58cf6160c3 100644
---
a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
+++
b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureConstants.java
@@ -119,5 +119,10 @@ public final class AzureConstants {
*/
public static final String PRESIGNED_HTTP_UPLOAD_URI_DOMAIN_OVERRIDE =
"presignedHttpUploadURIDomainOverride";
+/**
+ * Property to enable/disable creation of reference secret on init.
+ */
+public static final String AZURE_REF_ON_INIT = "refOnInit";
+
private AzureConstants() { }
}
diff --git
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
index 9cac0cbcaa..d37c6c7237 100644
---
a/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
+++
b/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackendTest.java
@@ -32,6 +32,9 @@ import java.util.EnumSet;
import java.util.Properties;
import java.util.Set;
import java.util.stream.StreamSupport;
+
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
import org.jetbrains.annotations.NotNull;
import org.junit.After;
import org.junit.ClassRule;
@@ -44,6 +47,8 @@ import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.READ;
import static
com.microsoft.azure.storage.blob.SharedAccessBlobPermissions.WRITE;
import static java.util.stream.Collectors.toSet;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class AzureBlobStoreBackendTest {
@@ -129,6 +134,15 @@ public class AzureBlobStoreBackendTest {
assertReadAccessGranted(azureBlobStoreBackend,
ImmutableSet.of("file"));
}
+@Test
+public void initSecret() throws Exception {
+AzureBlobStoreBackend azureBlobStoreBackend = new
AzureBlobStoreBackend();
+
azureBlobStoreBackend.setProperties(getConfigurationWithConnectionString());
+
+azureBlobStoreBackend.init();
+assertReferenceSecret(azureBlobStoreBacke
[jackrabbit-oak] branch OAK-9807 created (now f640551c26)
This is an automated email from the ASF dual-hosted git repository. amitj pushed a change to branch OAK-9807 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git at f640551c26 OAK-9807: On DataStore initialization create the reference.key if absent This branch includes the following new commits: new f640551c26 OAK-9807: On DataStore initialization create the reference.key if absent The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[jackrabbit-oak] branch trunk updated: OAK-9814: Improvements in NodeState/VersionCopier for visibility of p… (#603)
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5e1f45c9e7 OAK-9814: Improvements in NodeState/VersionCopier for
visibility of p… (#603)
5e1f45c9e7 is described below
commit 5e1f45c9e75c8dd8141ee9da2e011124d28af892
Author: Amit Jain
AuthorDate: Wed Jun 29 09:42:35 2022 +0530
OAK-9814: Improvements in NodeState/VersionCopier for visibility of p…
(#603)
- Added an optional consumer to be notified of any added paths
- Create a new preserveOnTarget which does not delete any sub-paths on
target
- Rename consumer variable to make clear intention of consuming added paths
only
---
.../oak/plugins/migration/NodeStateCopier.java | 91 --
.../plugins/migration/version/VersionCopier.java | 25 +-
.../version/VersionCopyConfiguration.java | 12 ++-
.../migration/version/VersionableEditor.java | 3 +-
.../oak/plugins/migration/NodeStateCopierTest.java | 39 ++
.../migration/version/VersionCopierTest.java | 88 -
6 files changed, 227 insertions(+), 31 deletions(-)
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
index f7ba441963..cebe0e1dd3 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/NodeStateCopier.java
@@ -33,6 +33,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Set;
+import java.util.function.Consumer;
+
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.ImmutableSet.copyOf;
import static com.google.common.collect.ImmutableSet.of;
@@ -70,6 +72,9 @@ import static java.util.Collections.emptySet;
* Merge paths: if merge paths are set, any nodes matching or
* below the merged path will not be deleted from target, even if they
* are missing in (or excluded from) the source.
+ * Preserve on Target if set to true the nodes on target under the
included
+ * paths are not deleted and the merge paths property is ignored. If false
+ * the deletion rules default to the case for merge paths.
*/
public class NodeStateCopier {
@@ -87,18 +92,26 @@ public class NodeStateCopier {
private final boolean referenceableFrozenNodes;
+private final boolean preserveOnTarget;
+
+private final Consumer newNodesConsumer;
+
private NodeStateCopier(Set includePaths,
Set excludePaths,
Set fragmentPaths,
Set excludeFragments,
Set mergePaths,
-boolean referenceableFrozenNodes) {
+boolean referenceableFrozenNodes,
+boolean preserveOnTarget,
+Consumer newNodesConsumer) {
this.includePaths = includePaths;
this.excludePaths = excludePaths;
this.fragmentPaths = fragmentPaths;
this.excludeFragments = excludeFragments;
this.mergePaths = mergePaths;
this.referenceableFrozenNodes = referenceableFrozenNodes;
+this.preserveOnTarget = preserveOnTarget;
+this.newNodesConsumer = newNodesConsumer;
}
/**
@@ -164,7 +177,8 @@ public class NodeStateCopier {
final NodeState sourceState =
NodeStateUtils.getNode(wrappedSource, includePath);
if (sourceState.exists()) {
final NodeBuilder targetBuilder =
getChildNodeBuilder(targetRoot, includePath);
-hasChanges = copyNodeState(sourceState, targetBuilder,
includePath, this.mergePaths) || hasChanges;
+hasChanges = copyNodeState(sourceState, targetBuilder,
includePath, this.mergePaths,
+this.preserveOnTarget, this.newNodesConsumer) ||
hasChanges;
}
}
return hasChanges;
@@ -185,17 +199,20 @@ public class NodeStateCopier {
* @param currentPath The path of both the source and target arguments.
* @param mergePaths A Set of paths under which existing nodes should be
* preserved, even if the do not exist in the source.
+ * @param newNodesConsumer A newNodesConsumer for the node paths being
added
* @return An indication of whether there were changes or not.
*/
private static boolean copyNodeState(@NotNull final NodeState source,
@NotNull final NodeBuilder target,
- @NotNull final String currentPath,
@NotNull final Set mergePaths) {
+@NotNull final String currentPath, @NotNull final
[jackrabbit-oak] branch trunk updated: OAK-9813: [oak-run-commons] LoggingInitializer shutdownLogging should not shut down if not initialized
This is an automated email from the ASF dual-hosted git repository.
amitj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 43a8e603f0 OAK-9813: [oak-run-commons] LoggingInitializer
shutdownLogging should not shut down if not initialized
new 54a78e4ba4 Merge pull request #602 from amit-jain/OAK-9813
43a8e603f0 is described below
commit 43a8e603f077ed5f06fca335fb2dd73f7417b3e0
Author: amjain
AuthorDate: Wed Jun 22 18:59:04 2022 +0530
OAK-9813: [oak-run-commons] LoggingInitializer shutdownLogging should not
shut down if not initialized
Ignore if system prop logback.configurationFile set
---
.../org/apache/jackrabbit/oak/run/commons/LoggingInitializer.java | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/commons/LoggingInitializer.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/commons/LoggingInitializer.java
index 35f33be844..e3239fe808 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/commons/LoggingInitializer.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/commons/LoggingInitializer.java
@@ -77,7 +77,12 @@ public class LoggingInitializer {
log.info("Logs would be written to {}", new File(workDir,
logIdentifier + ".log"));
}
-public static void shutdownLogging(){
+public static void shutdownLogging() {
+//Only shutdown if init
+if (System.getProperty(ContextInitializer.CONFIG_FILE_PROPERTY) !=
null) {
+return;
+}
+
LoggerContext context = (LoggerContext)
LoggerFactory.getILoggerFactory();
context.stop();
}
svn commit: r1887918 - in /jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob: MarkSweepGarbageCollector.java datastore/DataStoreBlobStore.java
Author: amitj
Date: Mon Mar 22 07:46:35 2021
New Revision: 1887918
URL: http://svn.apache.org/viewvc?rev=1887918&view=rev
Log:
OAK-9390: Log the repositoryIds during DSGC and on datastore init
- Logging repositoryId on init, dsgc start and other ids retrieved during sweep
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1887918&r1=1887917&r2=1887918&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Mar 22 07:46:35 2021
@@ -349,7 +349,7 @@ public class MarkSweepGarbageCollector i
Stopwatch sw = Stopwatch.createStarted();
try {
-LOG.info("Starting Blob garbage collection with markOnly [{}]",
markOnly);
+LOG.info("Starting Blob garbage collection with markOnly [{}] for
repositoryId [{}]", markOnly, repoId);
long markStart = System.currentTimeMillis();
long markFinish;
@@ -833,14 +833,17 @@ public class MarkSweepGarbageCollector i
List refFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+LOG.info("References available {}", refFiles);
// Get all the repositories registered
List repoFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType());
-
+LOG.info("Repositories registered {}", repoFiles);
+
// Retrieve repos for which reference files have not been
created
Set unAvailRepos =
SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles);
+LOG.info("Repositories with unavailable references {}",
unAvailRepos);
Set notOldRefs = Collections.EMPTY_SET;
long retentionTime = clock.getTime() - maxLastModifiedInterval;
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1887918&r1=1887917&r2=1887918&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Mon Mar 22 07:46:35 2021
@@ -759,6 +759,7 @@ public class DataStoreBlobStore
this.repositoryId = repositoryId;
addMetadataRecord(new ByteArrayInputStream(new byte[0]),
SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY.getNameFromId(repositoryId));
+log.info("repositoryId registered in blobstore - [{}]", repositoryId);
}
@Override
svn commit: r1879390 - in /jackrabbit/oak/branches/1.22: ./ oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Author: amitj
Date: Wed Jul 1 06:09:32 2020
New Revision: 1879390
URL: http://svn.apache.org/viewvc?rev=1879390&view=rev
Log:
OAK-9128: Support s3 regions apart from default AWS regions
Merge r1879352 from trunk
Modified:
jackrabbit/oak/branches/1.22/ (props changed)
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Propchange: jackrabbit/oak/branches/1.22/
--
Merged /jackrabbit/oak/trunk:r1879352
Modified:
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1879390&r1=1879389&r2=1879390&view=diff
==
---
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Wed Jul 1 06:09:32 2020
@@ -176,28 +176,22 @@ public class S3Backend extends AbstractS
}
}
String region = properties.getProperty(S3Constants.S3_REGION);
-Region s3Region;
+
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region =
Regions.getCurrentRegion();
if (ec2Region != null) {
-s3Region = Region.fromValue(ec2Region.getName());
+region = ec2Region.getName();
} else {
throw new AmazonClientException(
"parameter ["
+ S3Constants.S3_REGION
+ "] not configured and cannot be derived
from environment");
}
-} else {
-if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
-s3Region = Region.US_Standard;
-} else if (Region.EU_Ireland.toString().equals(region)) {
-s3Region = Region.EU_Ireland;
-} else {
-s3Region = Region.fromValue(region);
-}
+} else if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
+region = Region.US_Standard.toString();
}
-createBucketIfNeeded(s3Region);
+createBucketIfNeeded(region);
int writeThreads = 10;
String writeThreadsStr =
properties.getProperty(S3Constants.S3_WRITE_THREADS);
@@ -265,24 +259,24 @@ public class S3Backend extends AbstractS
}
}
-private void createBucketIfNeeded(@NotNull final Region s3Region) {
+private void createBucketIfNeeded(final String region) {
try {
if (!s3service.doesBucketExist(bucket)) {
-CreateBucketRequest req = new CreateBucketRequest(bucket,
s3Region);
+CreateBucketRequest req = new CreateBucketRequest(bucket,
region);
s3service.createBucket(req);
if (Utils.waitForBucket(s3service, bucket)) {
LOG.error("Bucket [{}] does not exist in [{}] and was not
automatically created",
-bucket, s3Region.name());
+bucket, region);
return;
}
-LOG.info("Created bucket [{}] in [{}] ", bucket,
s3Region.name());
+LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
-LOG.info("Using bucket [{}] in [{}] ", bucket,
s3Region.name());
+LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
}
catch (SdkClientException awsException) {
LOG.error("Attempt to create S3 bucket [{}] in [{}] failed",
-bucket, s3Region.name(), awsException);
+bucket, region, awsException);
}
}
svn commit: r1879352 - /jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Author: amitj
Date: Tue Jun 30 06:15:12 2020
New Revision: 1879352
URL: http://svn.apache.org/viewvc?rev=1879352&view=rev
Log:
OAK-9128: Support s3 regions apart from default AWS regions
- Patch from Neeraj Garg
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1879352&r1=1879351&r2=1879352&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Tue Jun 30 06:15:12 2020
@@ -176,28 +176,22 @@ public class S3Backend extends AbstractS
}
}
String region = properties.getProperty(S3Constants.S3_REGION);
-Region s3Region;
+
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region =
Regions.getCurrentRegion();
if (ec2Region != null) {
-s3Region = Region.fromValue(ec2Region.getName());
+region = ec2Region.getName();
} else {
throw new AmazonClientException(
"parameter ["
+ S3Constants.S3_REGION
+ "] not configured and cannot be derived
from environment");
}
-} else {
-if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
-s3Region = Region.US_Standard;
-} else if (Region.EU_Ireland.toString().equals(region)) {
-s3Region = Region.EU_Ireland;
-} else {
-s3Region = Region.fromValue(region);
-}
+} else if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
+region = Region.US_Standard.toString();
}
-createBucketIfNeeded(s3Region);
+createBucketIfNeeded(region);
int writeThreads = 10;
String writeThreadsStr =
properties.getProperty(S3Constants.S3_WRITE_THREADS);
@@ -265,24 +259,24 @@ public class S3Backend extends AbstractS
}
}
-private void createBucketIfNeeded(@NotNull final Region s3Region) {
+private void createBucketIfNeeded(final String region) {
try {
if (!s3service.doesBucketExist(bucket)) {
-CreateBucketRequest req = new CreateBucketRequest(bucket,
s3Region);
+CreateBucketRequest req = new CreateBucketRequest(bucket,
region);
s3service.createBucket(req);
if (Utils.waitForBucket(s3service, bucket)) {
LOG.error("Bucket [{}] does not exist in [{}] and was not
automatically created",
-bucket, s3Region.name());
+bucket, region);
return;
}
-LOG.info("Created bucket [{}] in [{}] ", bucket,
s3Region.name());
+LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
-LOG.info("Using bucket [{}] in [{}] ", bucket,
s3Region.name());
+LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
}
catch (SdkClientException awsException) {
LOG.error("Attempt to create S3 bucket [{}] in [{}] failed",
-bucket, s3Region.name(), awsException);
+bucket, region, awsException);
}
}
svn commit: r1878112 - in /jackrabbit/oak/branches/1.8: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Author: amitj
Date: Tue May 26 07:28:12 2020
New Revision: 1878112
URL: http://svn.apache.org/viewvc?rev=1878112&view=rev
Log:
OAK-9092: Exception root cause message is swallowed
Merge r1878109 from trunk
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Propchange: jackrabbit/oak/branches/1.8/
--
Merged /jackrabbit/oak/trunk:r1878109
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1878112&r1=1878111&r2=1878112&view=diff
==
---
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Tue May 26 07:28:12 2020
@@ -225,7 +225,8 @@ public class FileCache extends AbstractC
}
cache.put(key, cached);
} catch (IOException e) {
-LOG.error("Exception adding id [{}] with file [{}] to cache", key,
file);
+LOG.error("Exception adding id [{}] with file [{}] to cache, root
cause: {}", key, file, e.getMessage());
+LOG.debug("Root cause", e);
}
}
svn commit: r1878111 - in /jackrabbit/oak/branches/1.22: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Author: amitj
Date: Tue May 26 06:52:26 2020
New Revision: 1878111
URL: http://svn.apache.org/viewvc?rev=1878111&view=rev
Log:
OAK-9092: Exception root cause message is swallowed
Merge r1878109 from trunk
Modified:
jackrabbit/oak/branches/1.22/ (props changed)
jackrabbit/oak/branches/1.22/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Propchange: jackrabbit/oak/branches/1.22/
--
Merged /jackrabbit/oak/trunk:r1878109
Modified:
jackrabbit/oak/branches/1.22/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1878111&r1=1878110&r2=1878111&view=diff
==
---
jackrabbit/oak/branches/1.22/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
(original)
+++
jackrabbit/oak/branches/1.22/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Tue May 26 06:52:26 2020
@@ -225,7 +225,8 @@ public class FileCache extends AbstractC
}
cache.put(key, cached);
} catch (IOException e) {
-LOG.error("Exception adding id [{}] with file [{}] to cache", key,
file);
+LOG.error("Exception adding id [{}] with file [{}] to cache, root
cause: {}", key, file, e.getMessage());
+LOG.debug("Root cause", e);
}
}
svn commit: r1878109 - /jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Author: amitj
Date: Tue May 26 06:43:52 2020
New Revision: 1878109
URL: http://svn.apache.org/viewvc?rev=1878109&view=rev
Log:
OAK-9092: Exception root cause message is swallowed
- Logging the message and the stack trace at DEBUG
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1878109&r1=1878108&r2=1878109&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Tue May 26 06:43:52 2020
@@ -229,7 +229,8 @@ public class FileCache extends AbstractC
}
cache.put(key, cached);
} catch (IOException e) {
-LOG.error("Exception adding id [{}] with file [{}] to cache", key,
file);
+LOG.error("Exception adding id [{}] with file [{}] to cache, root
cause: {}", key, file, e.getMessage());
+LOG.debug("Root cause", e);
}
}
svn commit: r1877944 - in /jackrabbit/oak/trunk: oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/ oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak
Author: amitj
Date: Wed May 20 09:47:11 2020
New Revision: 1877944
URL: http://svn.apache.org/viewvc?rev=1877944&view=rev
Log:
OAK-9063: Use a custom metadata property to manage lastModified in
AzureDataStore
- Use a custom metadata property for lastModified, added when adding blob
normal and metadata
- Fallback to azure managed lastModified in case the custom metadata property
not present for existing blobs
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDS.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDSWithSmallCache.java
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/test/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/TestAzureDsCacheOff.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/AbstractDataStoreTest.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java?rev=1877944&r1=1877943&r2=1877944&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java
Wed May 20 09:47:11 2020
@@ -99,6 +99,7 @@ public class AzureBlobStoreBackend exten
private static final String META_KEY_PREFIX = META_DIR_NAME + "/";
private static final String REF_KEY = "reference.key";
+private static final String LAST_MODIFIED_KEY = "lastModified";
private static final long BUFFERED_STREAM_THRESHHOLD = 1024 * 1024;
static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; //
10MB
@@ -266,6 +267,8 @@ public class AzureBlobStoreBackend exten
LOG.debug("Blob write started. identifier={} length={}", key, len);
CloudBlockBlob blob =
getAzureContainer().getBlockBlobReference(key);
if (!blob.exists()) {
+addLastModified(blob);
+
BlobRequestOptions options = new BlobRequestOptions();
options.setConcurrentRequestCount(concurrentRequestCount);
boolean useBufferedStream = len < BUFFERED_STREAM_THRESHHOLD;
@@ -289,16 +292,13 @@ public class AzureBlobStoreBackend exten
" new length=" + len +
" old length=" +
blob.getProperties().getLength());
}
-LOG.trace("Blob already exists. identifier={} lastModified={}",
key, blob.getProperties().getLastModified().getTime());
-blob.startCopy(blob);
-//TODO: better way of updating lastModified (use custom metadata?)
-if (!waitForCopy(blob)) {
-throw new DataStoreException(
-String.format("Cannot update lastModified for blob.
identifier=%s status=%s",
- key,
blob.getCopyState().getStatusDescription()));
-}
+
+LOG.trace("Blob already exists. identifier={} lastModified={}",
key, getLastModified(blob));
+addLastModified(blob);
+blob.uploadMetadata();
+
LOG.debug("Blob updated. identifier={} lastModified={}
duration={}", key,
- blob.getProperties().getLastModified().getTime(),
(System.currentTimeMillis() - start));
+ getLastModified(blob), (System.currentTimeMillis() -
start));
}
catch (StorageException e) {
LOG.info("Error writing blob. identifier={}", key, e);
@@ -307,9 +307,6 @@ public class AzureBlobStoreBackend exten
catch (URISyntaxException | IOException e) {
LOG.debug("Error writing blob. identifier={}", key, e);
throw new DataStoreException(String.format("Cannot write blob.
identifier=%s", key), e);
-} catch (InterruptedException e) {
-LOG.debug("Error writing blob. identifier={}", key, e);
-throw new DataStoreException(String.format("Cannot copy blob.
identifier=%s", key), e);
svn commit: r1877915 - /jackrabbit/oak/trunk/oak-upgrade/pom.xml
Author: amitj
Date: Tue May 19 10:12:37 2020
New Revision: 1877915
URL: http://svn.apache.org/viewvc?rev=1877915&view=rev
Log:
OAK-8827: AWS support for segment-tar
- Add dependency to oak-segment-remote for oak-upgrade
Modified:
jackrabbit/oak/trunk/oak-upgrade/pom.xml
Modified: jackrabbit/oak/trunk/oak-upgrade/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-upgrade/pom.xml?rev=1877915&r1=1877914&r2=1877915&view=diff
==
--- jackrabbit/oak/trunk/oak-upgrade/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-upgrade/pom.xml Tue May 19 10:12:37 2020
@@ -113,6 +113,11 @@
org.apache.jackrabbit
+ oak-segment-remote
+ ${project.version}
+
+
+ org.apache.jackrabbit
oak-segment-azure
${project.version}
svn commit: r1877058 - in /jackrabbit/oak/trunk: oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
Author: amitj Date: Mon Apr 27 10:00:47 2020 New Revision: 1877058 URL: http://svn.apache.org/viewvc?rev=1877058&view=rev Log: OAK-9040: Option to only sweep in BlobGC when all references aged sufficiently - Added an option for the --collect-garbage to sweep only if the earliest references from all registered repositories is older than max age. Added: jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/MemoryBlobStoreNodeStore.java (with props) jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/TimeLapsedDataStore.java (with props) Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java jackrabbit/oak/trunk/oak-run/README.md jackrabbit/oak/trunk/oak-run/pom.xml jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1877058&r1=1877057&r2=1877058&view=diff == --- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original) +++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Mon Apr 27 10:00:47 2020 @@ -37,6 +37,7 @@ import java.io.InputStreamReader; import java.io.LineNumberReader; import java.sql.Timestamp; import java.util.ArrayDeque; +import java.util.Collections; import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -78,6 +79,7 @@ import org.apache.jackrabbit.oak.plugins import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.apache.jackrabbit.oak.stats.Clock; import org.apache.jackrabbit.oak.stats.CounterStats; import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.jackrabbit.oak.stats.StatsOptions; @@ -122,6 +124,9 @@ public class MarkSweepGarbageCollector i /** Flag to enable low cost consistency check after DSGC */ private boolean checkConsistencyAfterGc; +/* Flag to stop sweep if references not old enough */ +private final boolean sweepIfRefsPastRetention; + /** Helper class to mark blob references which **/ private final BlobReferenceRetriever marker; @@ -150,6 +155,8 @@ public class MarkSweepGarbageCollector i private boolean traceOutput; +private Clock clock; + /** * Creates an instance of MarkSweepGarbageCollector * @@ -174,6 +181,7 @@ public class MarkSweepGarbageCollector i int batchCount, long maxLastModifiedInterval, boolean checkConsistencyAfterGc, +boolean sweepIfRefsPastRetention, @Nullable String repositoryId, @Nullable Whiteboard whiteboard, @Nullable StatisticsProvider statisticsProvider) @@ -181,6 +189,7 @@ public class MarkSweepGarbageCollector i this.executor = executor; this.blobStore = blobStore; this.checkConsistencyAfterGc = checkConsistencyAfterGc; +this.sweepIfRefsPastRetention = sweepIfRefsPastRetention; checkNotNull(blobStore, "BlobStore cannot be null"); this.marker = marker; this.batchCount = batchCount; @@ -201,6 +210,7 @@ public class MarkSweepGarbageCollector i this.consistencyStats = new GarbageCollectionOperationStats(statisticsProvider, GarbageCollectionOperationStats.CONSISTENCY_NAME); this.consistencyStatsCollector = consistencyStats.getCollector(); +this.clock = Clock.SIMPLE; } public MarkSweepGarbageCollector(
svn commit: r1877059 - /jackrabbit/oak/trunk/oak-run/README.md
Author: amitj Date: Mon Apr 27 10:00:56 2020 New Revision: 1877059 URL: http://svn.apache.org/viewvc?rev=1877059&view=rev Log: OAK-9030: Retrieve datastore metadata using the DataStore command - documentation for --get-metadata datastore command Modified: jackrabbit/oak/trunk/oak-run/README.md Modified: jackrabbit/oak/trunk/oak-run/README.md URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1877059&r1=1877058&r2=1877059&view=diff == --- jackrabbit/oak/trunk/oak-run/README.md (original) +++ jackrabbit/oak/trunk/oak-run/README.md Mon Apr 27 10:00:56 2020 @@ -565,6 +565,8 @@ The following operations are available: --check-consistency- List all the missing blobs by doing a consistency check. --dump-ref - List all the blob references in the node store --dump-id - List all the ids in the data store +--get-metadata - Retrieves a machine readable format GC datastore metadata + e.g. |||[*-] * for local repo id The following options are available:
svn commit: r1877057 - in /jackrabbit/oak/trunk/oak-run/src: main/java/org/apache/jackrabbit/oak/run/ test/java/org/apache/jackrabbit/oak/run/
Author: amitj
Date: Mon Apr 27 10:00:22 2020
New Revision: 1877057
URL: http://svn.apache.org/viewvc?rev=1877057&view=rev
Log:
OAK-9030: Retrieve datastore metadata using the DataStore command
- Added an option get-metadata for the datastore which writes out all the
repository ids, earliest mark start timestamp secs, earliest mark refs finish
timestamp secs, */-
Added:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandMetadataTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java?rev=1877057&r1=1877056&r2=1877057&view=diff
==
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
Mon Apr 27 10:00:22 2020
@@ -20,6 +20,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -28,13 +29,17 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.TimeZone;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
+import java.util.stream.Collectors;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
import com.google.common.io.Closer;
import com.google.common.io.Files;
@@ -42,9 +47,11 @@ import joptsimple.OptionParser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.commons.io.BurnOnCloseFileIterator;
@@ -52,6 +59,8 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
+import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import
org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions;
@@ -103,15 +112,13 @@ public class DataStoreCommand implements
private DataStoreOptions dataStoreOpts;
private static final Comparator idComparator = new
Comparator() {
-@Override
-public int compare(String s1, String s2) {
+@Override public int compare(String s1, String s2) {
return s1.split(DELIM)[0].compareTo(s2.split(DELIM)[0]);
}
};
-@Override
-public void execute(String... args) throws Exception {
+@Override public void execute(String... args) throws Exception {
OptionParser parser = new OptionParser();
opts = new Options();
@@ -172,7 +179,7 @@ public class DataStoreCommand implements
return true;
}
-private void execute(NodeStoreFixture fixture, DataStoreOptions
dataStoreOpts, Options opts, Closer closer)
+private void execute(NodeStoreFixture fixture, DataStoreOptions
dataStoreOpts, Options opts, Closer closer)
throws Exception {
final BlobStoreOptions optionBean =
opts.getOptionBean(BlobStoreOptions.class);
@@ -190,10 +197,10 @@ public class DataStoreCommand implements
BlobReferenceRetriever retriever = getRetriever(fixture,
dataStoreOpts, opts);
retriever.collectReferences(new ReferenceCollector() {
-@Override
-public void addReferen
svn commit: r1874995 - in /jackrabbit/oak/branches/1.22: ./ oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/ oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/
Author: amitj
Date: Mon Mar 9 06:54:27 2020
New Revision: 1874995
URL: http://svn.apache.org/viewvc?rev=1874995&view=rev
Log:
OAK-8494: Support AWS Key Managed Service (SSE-KMS)
Merge r1874153 from trunk
Added:
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
- copied unchanged from r1874153,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
- copied unchanged from r1874153,
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
jackrabbit/oak/branches/1.22/oak-doc/src/site/markdown/features/direct-binary-access-upload-file.md
- copied unchanged from r1874153,
jackrabbit/oak/trunk/oak-doc/src/site/markdown/features/direct-binary-access-upload-file.md
Modified:
jackrabbit/oak/branches/1.22/ (props changed)
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/test/resources/aws.properties
jackrabbit/oak/branches/1.22/oak-doc/src/site/site.xml
Propchange: jackrabbit/oak/branches/1.22/
--
Merged /jackrabbit/oak/trunk:r1874153
Modified:
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==
---
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Mon Mar 9 06:54:27 2020
@@ -84,6 +84,7 @@ import com.google.common.collect.Abstrac
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
+import org.apache.http.protocol.HTTP;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
@@ -865,7 +866,7 @@ public class S3Backend extends AbstractS
else {
// multi-part
InitiateMultipartUploadRequest req = new
InitiateMultipartUploadRequest(bucket, blobId);
-InitiateMultipartUploadResult res =
s3service.initiateMultipartUpload(req);
+InitiateMultipartUploadResult res =
s3service.initiateMultipartUpload(s3ReqDecorator.decorate(req));
uploadId = res.getUploadId();
long numParts;
@@ -1024,6 +1025,10 @@ public class S3Backend extends AbstractS
.withMethod(method)
.withExpiration(expiration);
+if (method != HttpMethod.GET) {
+ request = s3ReqDecorator.decorate(request);
+}
+
for (Map.Entry e : reqParams.entrySet()) {
request.addRequestParameter(e.getKey(), e.getValue());
}
Modified:
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java?rev=1874995&r1=1874994&r2=1874995&view=diff
==
---
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
(original)
+++
jackrabbit/oak/branches/1.22/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
Mon Mar 9 06:54:27 2020
@@ -103,6 +103,21 @@ public final class S3Constants {
public static final String S3_ENCRYPTION_SSE_S3 = "SSE_S3";
/**
+ * Constant to set SSE_KMS encrypt
svn commit: r1874237 - in /jackrabbit/oak/trunk: oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
Author: amitj
Date: Thu Feb 20 05:43:41 2020
New Revision: 1874237
URL: http://svn.apache.org/viewvc?rev=1874237&view=rev
Log:
OAK-8903: Support for removing repository markers for transient repositories
- System flag oak.datastore.sharedTransient which if set to true will force
deletion of the repository marker from the datastore on close()
- Added 2 new default setter/getter methods for repository ids in the
SharedDataStore interface with the only implementation needed in the
DataStoreBlobStore (which is the glue between BlobStore & DataStore)
- The construction code (*NodeStoreService & tests) don't need to add the
metadata file to the DataStore directly and can simply call the
setRepositoryId() on the DataStoreBlobStore.
- Additional tests and changes to existing test to change the call above in 3.
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/AbstractBlobGCRegistrationTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java?rev=1874237&r1=1874236&r2=1874237&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
Thu Feb 20 05:43:41 2020
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.jetbrains.annotations.Nullable;
/**
* Interface to be implemented by a shared data store.
@@ -115,5 +116,21 @@ public interface SharedDataStore {
* @return the type
*/
Type getType();
+
+/**
+ * Returns the repository id (identifier for the repository in the
DataStore)
+ * @return repository id
+ */
+@Nullable
+default String getRepositoryId() {
+return null;
+}
+
+/**
+ * Sets the repository id to identify repository in the DataStore
+ * @param repositoryId
+ * @throws DataStoreException
+ */
+default void setRepositoryId(String repositoryId) throws
DataStoreException {}
}
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1874237&r1=1874236&r2=1874237&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Thu Feb 20 05:43:41 2020
@@ -44,6 +44,7 @@ import javax.jcr.RepositoryException;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
import com.google.common.cache.LoadingCache;
import
svn commit: r1874154 - /jackrabbit/oak/trunk/oak-run/pom.xml
Author: amitj Date: Tue Feb 18 05:42:55 2020 New Revision: 1874154 URL: http://svn.apache.org/viewvc?rev=1874154&view=rev Log: OAK-8783: Merge index definitions - Ignore merge*.txt test resource files for rat check Modified: jackrabbit/oak/trunk/oak-run/pom.xml Modified: jackrabbit/oak/trunk/oak-run/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/pom.xml?rev=1874154&r1=1874153&r2=1874154&view=diff == --- jackrabbit/oak/trunk/oak-run/pom.xml (original) +++ jackrabbit/oak/trunk/oak-run/pom.xml Tue Feb 18 05:42:55 2020 @@ -184,6 +184,16 @@ true + +org.apache.rat +apache-rat-plugin + + +**/merge.txt +**/mergeIndexes.txt + + +
svn commit: r1874153 - in /jackrabbit/oak/trunk: oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/ oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/ oak-blob-cl
Author: amitj
Date: Tue Feb 18 05:22:52 2020
New Revision: 1874153
URL: http://svn.apache.org/viewvc?rev=1874153&view=rev
Log:
OAK-8494 - Support AWS Key Managed Service (SSE-KMS)
- Patch from Ankita Agarwal
Added:
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMS.java
(with props)
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSEKMSwithKey.java
(with props)
jackrabbit/oak/trunk/oak-doc/src/site/markdown/features/direct-binary-access-upload-file.md
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DSWithSSES3.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
jackrabbit/oak/trunk/oak-blob-cloud/src/test/resources/aws.properties
jackrabbit/oak/trunk/oak-doc/src/site/site.xml
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java?rev=1874153&r1=1874152&r2=1874153&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java
Tue Feb 18 05:22:52 2020
@@ -84,6 +84,7 @@ import com.google.common.collect.Abstrac
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
+import org.apache.http.protocol.HTTP;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
@@ -865,7 +866,7 @@ public class S3Backend extends AbstractS
else {
// multi-part
InitiateMultipartUploadRequest req = new
InitiateMultipartUploadRequest(bucket, blobId);
-InitiateMultipartUploadResult res =
s3service.initiateMultipartUpload(req);
+InitiateMultipartUploadResult res =
s3service.initiateMultipartUpload(s3ReqDecorator.decorate(req));
uploadId = res.getUploadId();
long numParts;
@@ -1024,6 +1025,10 @@ public class S3Backend extends AbstractS
.withMethod(method)
.withExpiration(expiration);
+if (method != HttpMethod.GET) {
+ request = s3ReqDecorator.decorate(request);
+}
+
for (Map.Entry e : reqParams.entrySet()) {
request.addRequestParameter(e.getKey(), e.getValue());
}
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java?rev=1874153&r1=1874152&r2=1874153&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
Tue Feb 18 05:22:52 2020
@@ -103,6 +103,21 @@ public final class S3Constants {
public static final String S3_ENCRYPTION_SSE_S3 = "SSE_S3";
/**
+ * Constant to set SSE_KMS encryption.
+ */
+public static final String S3_ENCRYPTION_SSE_KMS = "SSE_KMS";
+
+/**
+ * Constant to set keyID for SSE_KMS encryption.
+ */
+public static final String S3_SSE_KMS_KEYID = "kmsKeyId";
+
+/**
+ * Constant to set S3 signature for SSE_KMS encryption.
+ */
+public static final String S3_SIG_V4 = "AWSS3V4SignerType";
+
+/**
* Constant to set proxy host.
*/
public static final String PROXY_HOST = "proxyHost";
Modified:
jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3RequestDecorator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blo
svn commit: r1873281 - in /jackrabbit/oak/branches/1.10: ./ oak-blob-cloud/pom.xml oak-run/pom.xml
Author: amitj Date: Wed Jan 29 05:56:52 2020 New Revision: 1873281 URL: http://svn.apache.org/viewvc?rev=1873281&view=rev Log: OAK-8875: Update aws-java-sdk-osgi version to 1.11.700 in Oak-blob-cloud - Merge r1873279 from trunk Modified: jackrabbit/oak/branches/1.10/ (props changed) jackrabbit/oak/branches/1.10/oak-blob-cloud/pom.xml jackrabbit/oak/branches/1.10/oak-run/pom.xml Propchange: jackrabbit/oak/branches/1.10/ -- Merged /jackrabbit/oak/trunk:r1873279 Modified: jackrabbit/oak/branches/1.10/oak-blob-cloud/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-cloud/pom.xml?rev=1873281&r1=1873280&r2=1873281&view=diff == --- jackrabbit/oak/branches/1.10/oak-blob-cloud/pom.xml (original) +++ jackrabbit/oak/branches/1.10/oak-blob-cloud/pom.xml Wed Jan 29 05:56:52 2020 @@ -32,7 +32,7 @@ 4.1.17.Final -1.11.330 +1.11.700 Modified: jackrabbit/oak/branches/1.10/oak-run/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-run/pom.xml?rev=1873281&r1=1873280&r2=1873281&view=diff == --- jackrabbit/oak/branches/1.10/oak-run/pom.xml (original) +++ jackrabbit/oak/branches/1.10/oak-run/pom.xml Wed Jan 29 05:56:52 2020 @@ -35,6 +35,7 @@ 2.4.17 -5100 +5200
svn commit: r1873279 - in /jackrabbit/oak/trunk: oak-blob-cloud/pom.xml oak-run/pom.xml
Author: amitj Date: Wed Jan 29 05:27:31 2020 New Revision: 1873279 URL: http://svn.apache.org/viewvc?rev=1873279&view=rev Log: OAK-8875: Update aws-java-sdk-osgi version to 1.11.700 in Oak-blob-cloud - Updated to 1.11.700 - Bumping up limits for oak-run jar size to 52 MB Modified: jackrabbit/oak/trunk/oak-blob-cloud/pom.xml jackrabbit/oak/trunk/oak-run/pom.xml Modified: jackrabbit/oak/trunk/oak-blob-cloud/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-cloud/pom.xml?rev=1873279&r1=1873278&r2=1873279&view=diff == --- jackrabbit/oak/trunk/oak-blob-cloud/pom.xml (original) +++ jackrabbit/oak/trunk/oak-blob-cloud/pom.xml Wed Jan 29 05:27:31 2020 @@ -32,7 +32,7 @@ 4.1.17.Final -1.11.330 +1.11.700 Modified: jackrabbit/oak/trunk/oak-run/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/pom.xml?rev=1873279&r1=1873278&r2=1873279&view=diff == --- jackrabbit/oak/trunk/oak-run/pom.xml (original) +++ jackrabbit/oak/trunk/oak-run/pom.xml Wed Jan 29 05:27:31 2020 @@ -35,6 +35,7 @@ 2.4.17 -5100 +5200
svn commit: r1873199 - in /jackrabbit/oak/branches/1.8: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak
Author: amitj
Date: Mon Jan 27 09:37:06 2020
New Revision: 1873199
URL: http://svn.apache.org/viewvc?rev=1873199&view=rev
Log:
OAK-8807: Update FileCache to use non-cache segment blocking method
- Merge r1870525 from trunk
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStoreCacheTest.java
Propchange: jackrabbit/oak/branches/1.8/
--
Merged /jackrabbit/oak/trunk:r1870525
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1873199&r1=1873198&r2=1873199&view=diff
==
---
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Mon Jan 27 09:37:06 2020
@@ -79,6 +79,8 @@ public class FileCache extends AbstractC
private ExecutorService executor;
+private CacheLoader cacheLoader;
+
/**
* Convert the size calculation to KB to support max file size of 2 TB
*/
@@ -103,6 +105,30 @@ public class FileCache extends AbstractC
/* convert to 4 KB block */
long size = Math.round(maxSize / (1024L * 4));
+cacheLoader = new CacheLoader() {
+@Override public File load(String key) throws Exception {
+// Fetch from local cache directory and if not found load from
backend
+File cachedFile = DataStoreCacheUtils.getFile(key, cacheRoot);
+if (cachedFile.exists()) {
+return cachedFile;
+} else {
+InputStream is = null;
+boolean threw = true;
+try {
+is = loader.load(key);
+copyInputStreamToFile(is, cachedFile);
+threw = false;
+} catch (Exception e) {
+LOG.warn("Error reading object for id [{}] from
backend", key, e);
+throw e;
+} finally {
+Closeables.close(is, threw);
+}
+return cachedFile;
+}
+}
+};
+
cache = new CacheLIRS.Builder()
.maximumWeight(size)
.recordStats()
@@ -122,30 +148,8 @@ public class FileCache extends AbstractC
LOG.info("Cached file deletion failed after eviction",
e);
}
}})
-.build(new CacheLoader() {
-@Override
-public File load(String key) throws Exception {
-// Fetch from local cache directory and if not found load
from backend
-File cachedFile = DataStoreCacheUtils.getFile(key,
cacheRoot);
-if (cachedFile.exists()) {
-return cachedFile;
-} else {
-InputStream is = null;
-boolean threw = true;
-try {
-is = loader.load(key);
-copyInputStreamToFile(is, cachedFile);
-threw = false;
-} catch (Exception e) {
-LOG.warn("Error reading object for id [{}] from
backend", key, e);
-throw e;
-} finally {
-Closeables.close(is, threw);
-}
-return cachedFile;
-}
-}
-});
+.build();
+
this.cacheStats =
new FileCacheStats(cache, weigher, memWeigher, maxSize);
@@ -254,7 +258,7 @@ public class FileCache extends AbstractC
public File get(String key) throws IOException {
try {
// get from cache and download if not available
-return cache.get(key);
+return cache.get(key, () -> cacheLoader.load(key));
} catch (ExecutionException e) {
LOG.error("Error loading [{}] from cache", key);
throw new IOException(e);
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStoreCacheTest.java
URL:
http:/
svn commit: r1873198 - in /jackrabbit/oak/branches/1.10: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java oak-blob-plugins/src/test/java/org/apache/jackrabbit/oa
Author: amitj
Date: Mon Jan 27 09:17:04 2020
New Revision: 1873198
URL: http://svn.apache.org/viewvc?rev=1873198&view=rev
Log:
OAK-8807: Update FileCache to use non-cache segment blocking method
- Merge r1870525 from trunk
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStoreCacheTest.java
Propchange: jackrabbit/oak/branches/1.10/
--
Merged /jackrabbit/oak/trunk:r1870374,1870525
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1873198&r1=1873197&r2=1873198&view=diff
==
---
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Mon Jan 27 09:17:04 2020
@@ -79,6 +79,8 @@ public class FileCache extends AbstractC
private ExecutorService executor;
+private CacheLoader cacheLoader;
+
/**
* Convert the size calculation to KB to support max file size of 2 TB
*/
@@ -103,6 +105,30 @@ public class FileCache extends AbstractC
/* convert to 4 KB block */
long size = Math.round(maxSize / (1024L * 4));
+cacheLoader = new CacheLoader() {
+@Override public File load(String key) throws Exception {
+// Fetch from local cache directory and if not found load from
backend
+File cachedFile = DataStoreCacheUtils.getFile(key, cacheRoot);
+if (cachedFile.exists()) {
+return cachedFile;
+} else {
+InputStream is = null;
+boolean threw = true;
+try {
+is = loader.load(key);
+copyInputStreamToFile(is, cachedFile);
+threw = false;
+} catch (Exception e) {
+LOG.warn("Error reading object for id [{}] from
backend", key, e);
+throw e;
+} finally {
+Closeables.close(is, threw);
+}
+return cachedFile;
+}
+}
+};
+
cache = new CacheLIRS.Builder()
.maximumWeight(size)
.recordStats()
@@ -122,30 +148,8 @@ public class FileCache extends AbstractC
LOG.info("Cached file deletion failed after eviction",
e);
}
}})
-.build(new CacheLoader() {
-@Override
-public File load(String key) throws Exception {
-// Fetch from local cache directory and if not found load
from backend
-File cachedFile = DataStoreCacheUtils.getFile(key,
cacheRoot);
-if (cachedFile.exists()) {
-return cachedFile;
-} else {
-InputStream is = null;
-boolean threw = true;
-try {
-is = loader.load(key);
-copyInputStreamToFile(is, cachedFile);
-threw = false;
-} catch (Exception e) {
-LOG.warn("Error reading object for id [{}] from
backend", key, e);
-throw e;
-} finally {
-Closeables.close(is, threw);
-}
-return cachedFile;
-}
-}
-});
+.build();
+
this.cacheStats =
new FileCacheStats(cache, weigher, memWeigher, maxSize);
@@ -254,7 +258,7 @@ public class FileCache extends AbstractC
public File get(String key) throws IOException {
try {
// get from cache and download if not available
-return cache.get(key);
+return cache.get(key, () -> cacheLoader.load(key));
} catch (ExecutionException e) {
LOG.error("Error loading [{}] from cache", key);
throw new IOException(e);
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStoreCacheTest.java
svn commit: r1872908 - /jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Author: amitj
Date: Fri Jan 17 10:04:49 2020
New Revision: 1872908
URL: http://svn.apache.org/viewvc?rev=1872908&view=rev
Log:
OAK-8781: Add debug logs before and after cache purge to get the uploadCache
size info
Merge r1870374 from trunk
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java?rev=1872908&r1=1872907&r2=1872908&view=diff
==
---
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Fri Jan 17 10:04:49 2020
@@ -442,7 +442,7 @@ public class UploadStagingCache implemen
* Removes all cached from attic
*/
private void remove() {
-LOG.info("Starting purge of uploaded files");
+LOG.info("Starting purge of uploaded files, current size [{}]",
humanReadableByteCount(currentSize.get()));
Iterator iterator = attic.keySet().iterator();
int count = 0;
@@ -465,7 +465,7 @@ public class UploadStagingCache implemen
}
}
-LOG.info("Finished removal of [{}] files", count);
+LOG.info("Finished removal of [{}] files, current size [{}]", count,
humanReadableByteCount(currentSize.get()));
}
/**
svn commit: r1872907 - in /jackrabbit/oak/branches/1.10: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java oak-blob-plugins/src/test/java/org/apache/jac
Author: amitj
Date: Fri Jan 17 10:01:36 2020
New Revision: 1872907
URL: http://svn.apache.org/viewvc?rev=1872907&view=rev
Log:
OAK-8780: Remove the synchronized block on the FSBackend
Merge r1872803, r1872856 from trunk
Added:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
- copied, changed from r1872803,
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
Propchange: jackrabbit/oak/branches/1.10/
--
Merged /jackrabbit/oak/trunk:r1872803,1872856
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java?rev=1872907&r1=1872906&r2=1872907&view=diff
==
---
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
Fri Jan 17 10:01:36 2020
@@ -43,6 +43,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.LazyFileInputStream;
import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.apache.jackrabbit.util.TransientFileFactory;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,20 +105,38 @@ public class FSBackend extends AbstractS
@Override
public void write(DataIdentifier identifier, File file) throws
DataStoreException {
+TransientFileFactory fileFactory = TransientFileFactory.getInstance();
+File tmpFile = null;
+
File dest = getFile(identifier, fsPathDir);
-synchronized (this) {
-if (dest.exists()) {
-long now = System.currentTimeMillis();
-if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
-setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+if (dest.exists()) {
+long now = System.currentTimeMillis();
+if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
+setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+}
+} else {
+try {
+tmpFile = fileFactory.createTransientFile("fsbackend", null,
fsPathDir);
+FileUtils.copyFile(file, tmpFile);
+
+File parent = dest.getParentFile();
+parent.mkdirs();
+synchronized (this) {
+if (tmpFile.renameTo(dest)) {
+// no longer need to delete the temporary file
+tmpFile = null;
+} else {
+throw new IOException(
+"Can not rename " + tmpFile.getAbsolutePath()
++ " to " + dest.getAbsolutePath()
++ " (media read only?)");
+}
}
-} else {
-try {
-FileUtils.copyFile(file, dest);
-} catch (IOException ie) {
-LOG.error("failed to copy [{}] to [{}]",
file.getAbsolutePath(),
-dest.getAbsolutePath());
-throw new DataStoreException("Not able to write file [" +
identifier + "]", ie);
+} catch (IOException e) {
+throw new DataStoreException("Could not add record", e);
+} finally {
+if (tmpFile != null) {
+tmpFile.delete();
}
}
}
Copied:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
(from r1872803,
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java?p2=jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java&p1=jackrabbit/oak/trunk/oak-blob-plu
svn commit: r1872906 - in /jackrabbit/oak/branches/1.8: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plu
Author: amitj
Date: Fri Jan 17 09:42:11 2020
New Revision: 1872906
URL: http://svn.apache.org/viewvc?rev=1872906&view=rev
Log:
OAK-8780: Remove the synchronized block on the FSBackend
Merge r1872803, r1872856 from trunk
Added:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
- copied, changed from r1872803,
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
Propchange: jackrabbit/oak/branches/1.8/
--
Merged /jackrabbit/oak/trunk:r1872803,1872856
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java?rev=1872906&r1=1872905&r2=1872906&view=diff
==
---
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
Fri Jan 17 09:42:11 2020
@@ -42,6 +42,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.LazyFileInputStream;
import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.apache.jackrabbit.util.TransientFileFactory;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,20 +103,38 @@ public class FSBackend extends AbstractS
@Override
public void write(DataIdentifier identifier, File file) throws
DataStoreException {
+TransientFileFactory fileFactory = TransientFileFactory.getInstance();
+File tmpFile = null;
+
File dest = getFile(identifier, fsPathDir);
-synchronized (this) {
-if (dest.exists()) {
-long now = System.currentTimeMillis();
-if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
-setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+if (dest.exists()) {
+long now = System.currentTimeMillis();
+if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
+setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+}
+} else {
+try {
+tmpFile = fileFactory.createTransientFile("fsbackend", null,
fsPathDir);
+FileUtils.copyFile(file, tmpFile);
+
+File parent = dest.getParentFile();
+parent.mkdirs();
+synchronized (this) {
+if (tmpFile.renameTo(dest)) {
+// no longer need to delete the temporary file
+tmpFile = null;
+} else {
+throw new IOException(
+"Can not rename " + tmpFile.getAbsolutePath()
++ " to " + dest.getAbsolutePath()
++ " (media read only?)");
+}
}
-} else {
-try {
-FileUtils.copyFile(file, dest);
-} catch (IOException ie) {
-LOG.error("failed to copy [{}] to [{}]",
file.getAbsolutePath(),
-dest.getAbsolutePath());
-throw new DataStoreException("Not able to write file [" +
identifier + "]", ie);
+} catch (IOException e) {
+throw new DataStoreException("Could not add record", e);
+} finally {
+if (tmpFile != null) {
+tmpFile.delete();
}
}
}
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java?rev=1872906&r1=1872905&r2=1872906&view=diff
==
---
jackrabbit/oak/branches/
svn commit: r1872905 - in /jackrabbit/oak/branches/1.8: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Author: amitj
Date: Fri Jan 17 09:31:16 2020
New Revision: 1872905
URL: http://svn.apache.org/viewvc?rev=1872905&view=rev
Log:
OAK-8781: Add debug logs before and after cache purge to get the uploadCache
size info
Merge r1870374 from trunk
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Propchange: jackrabbit/oak/branches/1.8/
--
Merged /jackrabbit/oak/trunk:r1870374
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java?rev=1872905&r1=1872904&r2=1872905&view=diff
==
---
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Fri Jan 17 09:31:16 2020
@@ -442,7 +442,7 @@ public class UploadStagingCache implemen
* Removes all cached from attic
*/
private void remove() {
-LOG.info("Starting purge of uploaded files");
+LOG.info("Starting purge of uploaded files, current size [{}]",
humanReadableByteCount(currentSize.get()));
Iterator iterator = attic.keySet().iterator();
int count = 0;
@@ -465,7 +465,7 @@ public class UploadStagingCache implemen
}
}
-LOG.info("Finished removal of [{}] files", count);
+LOG.info("Finished removal of [{}] files, current size [{}]", count,
humanReadableByteCount(currentSize.get()));
}
/**
svn commit: r1872856 - /jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
Author: amitj
Date: Thu Jan 16 05:24:40 2020
New Revision: 1872856
URL: http://svn.apache.org/viewvc?rev=1872856&view=rev
Log:
OAK-8780 - Remove the synchronized block on the FSBackend
Fix the failing tests
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java?rev=1872856&r1=1872855&r2=1872856&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
Thu Jan 16 05:24:40 2020
@@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -226,11 +227,10 @@ public class FSBackendIT {
writeLatch.await();
backend.write(new DataIdentifier("ID" + seed), f);
LOG.info("Added file to backend");
-return seed;
} catch (Exception e) {
LOG.error("Error adding file to backend", e);
-throw e;
}
+return seed;
});
futures.add(future);
svn commit: r1872803 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java test/java/org/apache/jackrabbit/oak/plugins/blob/datasto
Author: amitj
Date: Wed Jan 15 05:01:02 2020
New Revision: 1872803
URL: http://svn.apache.org/viewvc?rev=1872803&view=rev
Log:
OAK-8780 - Remove the synchronized block on the FSBackend
Slightly modified patch from Ankita Agarwal
Moved the copy of file to a temp file outside the synchronized block
Added:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java?rev=1872803&r1=1872802&r2=1872803&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackend.java
Wed Jan 15 05:01:02 2020
@@ -43,6 +43,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.LazyFileInputStream;
import org.apache.jackrabbit.oak.spi.blob.AbstractDataRecord;
import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.apache.jackrabbit.util.TransientFileFactory;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -104,20 +105,38 @@ public class FSBackend extends AbstractS
@Override
public void write(DataIdentifier identifier, File file) throws
DataStoreException {
+TransientFileFactory fileFactory = TransientFileFactory.getInstance();
+File tmpFile = null;
+
File dest = getFile(identifier, fsPathDir);
-synchronized (this) {
-if (dest.exists()) {
-long now = System.currentTimeMillis();
-if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
-setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+if (dest.exists()) {
+long now = System.currentTimeMillis();
+if (getLastModified(dest) < now + ACCESS_TIME_RESOLUTION) {
+setLastModified(dest, now + ACCESS_TIME_RESOLUTION);
+}
+} else {
+try {
+tmpFile = fileFactory.createTransientFile("fsbackend", null,
fsPathDir);
+FileUtils.copyFile(file, tmpFile);
+
+File parent = dest.getParentFile();
+parent.mkdirs();
+synchronized (this) {
+if (tmpFile.renameTo(dest)) {
+// no longer need to delete the temporary file
+tmpFile = null;
+} else {
+throw new IOException(
+"Can not rename " + tmpFile.getAbsolutePath()
++ " to " + dest.getAbsolutePath()
++ " (media read only?)");
+}
}
-} else {
-try {
-FileUtils.copyFile(file, dest);
-} catch (IOException ie) {
-LOG.error("failed to copy [{}] to [{}]",
file.getAbsolutePath(),
-dest.getAbsolutePath());
-throw new DataStoreException("Not able to write file [" +
identifier + "]", ie);
+} catch (IOException e) {
+throw new DataStoreException("Could not add record", e);
+} finally {
+if (tmpFile != null) {
+tmpFile.delete();
}
}
}
Added:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java?rev=1872803&view=auto
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
(added)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/FSBackendIT.java
Wed Jan 15 05:01:02 2020
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
svn commit: r1870525 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStor
Author: amitj
Date: Thu Nov 28 04:26:38 2019
New Revision: 1870525
URL: http://svn.apache.org/viewvc?rev=1870525&view=rev
Log:
OAK-8807: Update FileCache to use non-cache segment blocking method
- Building the cache without the CacheLoader and using the get method with a
valueLoader
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStoreCacheTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1870525&r1=1870524&r2=1870525&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
Thu Nov 28 04:26:38 2019
@@ -79,6 +79,8 @@ public class FileCache extends AbstractC
private ExecutorService executor;
+private CacheLoader cacheLoader;
+
/**
* Convert the size calculation to KB to support max file size of 2 TB
*/
@@ -103,6 +105,30 @@ public class FileCache extends AbstractC
/* convert to 4 KB block */
long size = Math.round(maxSize / (1024L * 4));
+cacheLoader = new CacheLoader() {
+@Override public File load(String key) throws Exception {
+// Fetch from local cache directory and if not found load from
backend
+File cachedFile = DataStoreCacheUtils.getFile(key, cacheRoot);
+if (cachedFile.exists()) {
+return cachedFile;
+} else {
+InputStream is = null;
+boolean threw = true;
+try {
+is = loader.load(key);
+copyInputStreamToFile(is, cachedFile);
+threw = false;
+} catch (Exception e) {
+LOG.warn("Error reading object for id [{}] from
backend", key, e);
+throw e;
+} finally {
+Closeables.close(is, threw);
+}
+return cachedFile;
+}
+}
+};
+
cache = new CacheLIRS.Builder()
.maximumWeight(size)
.recordStats()
@@ -122,30 +148,8 @@ public class FileCache extends AbstractC
LOG.info("Cached file deletion failed after eviction",
e);
}
}})
-.build(new CacheLoader() {
-@Override
-public File load(String key) throws Exception {
-// Fetch from local cache directory and if not found load
from backend
-File cachedFile = DataStoreCacheUtils.getFile(key,
cacheRoot);
-if (cachedFile.exists()) {
-return cachedFile;
-} else {
-InputStream is = null;
-boolean threw = true;
-try {
-is = loader.load(key);
-copyInputStreamToFile(is, cachedFile);
-threw = false;
-} catch (Exception e) {
-LOG.warn("Error reading object for id [{}] from
backend", key, e);
-throw e;
-} finally {
-Closeables.close(is, threw);
-}
-return cachedFile;
-}
-}
-});
+.build();
+
this.cacheStats =
new FileCacheStats(cache, weigher, memWeigher, maxSize);
@@ -254,7 +258,7 @@ public class FileCache extends AbstractC
public File get(String key) throws IOException {
try {
// get from cache and download if not available
-return cache.get(key);
+return cache.get(key, () -> cacheLoader.load(key));
} catch (ExecutionException e) {
LOG.error("Error loading [{}] from cache", key);
throw new IOException(e);
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStoreCacheTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CompositeDataStor
svn commit: r1870374 - /jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Author: amitj
Date: Mon Nov 25 10:10:13 2019
New Revision: 1870374
URL: http://svn.apache.org/viewvc?rev=1870374&view=rev
Log:
OAK-8781: Add debug logs before and after cache purge to get the uploadCache
size info
- Added size in the logs when logging purge start and finish
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java?rev=1870374&r1=1870373&r2=1870374&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/UploadStagingCache.java
Mon Nov 25 10:10:13 2019
@@ -447,7 +447,7 @@ public class UploadStagingCache implemen
* Removes all cached from attic
*/
private void remove() {
-LOG.info("Starting purge of uploaded files");
+LOG.info("Starting purge of uploaded files, current size [{}]",
humanReadableByteCount(currentSize.get()));
Iterator iterator = attic.keySet().iterator();
int count = 0;
@@ -470,7 +470,7 @@ public class UploadStagingCache implemen
}
}
-LOG.info("Finished removal of [{}] files", count);
+LOG.info("Finished removal of [{}] files, current size [{}]", count,
humanReadableByteCount(currentSize.get()));
}
/**
svn commit: r1870373 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java test/java/org/apache/jackrabbit/oak/plugins/b
Author: amitj
Date: Mon Nov 25 10:10:03 2019
New Revision: 1870373
URL: http://svn.apache.org/viewvc?rev=1870373&view=rev
Log:
OAK-8786: Remove temporary file on upload immediately after a sync upload
- Delete temporary file after sync upload
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java?rev=1870373&r1=1870372&r2=1870373&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java
Mon Nov 25 10:10:03 2019
@@ -253,6 +253,10 @@ public abstract class AbstractSharedCach
LOG.info("Added blob [{}] to backend", identifier);
// offer to download cache
cache.getDownloadCache().put(identifier.toString(), tmpFile);
+
+// Attempt deleting the temporary file
+boolean deletedTemp = FileUtils.deleteQuietly(tmpFile);
+LOG.trace("Temporary file deleted {}", deletedTemp);
}
return getRecordIfStored(identifier);
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java?rev=1870373&r1=1870372&r2=1870373&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/CachingDataStoreTest.java
Mon Nov 25 10:10:03 2019
@@ -330,6 +330,30 @@ public class CachingDataStoreTest extend
}
/**
+ * Add sync and delete temp
+ * @throws Exception
+ */
+@Test
+public void syncAddTempDelete() throws Exception {
+LOG.info("Starting syncAddTempDelete");
+
+File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile());
+String id = getIdForInputStream(f);
+FileInputStream fin = new FileInputStream(f);
+closer.register(fin);
+
+DataRecord rec = dataStore.addRecord(fin, new
BlobOptions().setUpload(SYNCHRONOUS));
+assertEquals(id, rec.getIdentifier().toString());
+assertFile(rec.getStream(), f, folder);
+
+Collection files =
+FileUtils.listFiles(new File(dsPath, "tmp"),
FileFilterUtils.prefixFileFilter("upload"), null);
+assertEquals(0, files.size());
+
+LOG.info("Finished syncAddTempDelete");
+}
+
+/**
* {@link CompositeDataStoreCache#getIfPresent(String)} when no record.
*/
@Test
svn commit: r1868419 - in /jackrabbit/oak/trunk/oak-run/src: main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Author: amitj
Date: Mon Oct 14 04:01:05 2019
New Revision: 1868419
URL: http://svn.apache.org/viewvc?rev=1868419&view=rev
Log:
OAK-8697: Filter sensitive information when logging parameters for 'datastore'
command
Removed mongo URI and azure blob connection string from logging args
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java?rev=1868419&r1=1868418&r2=1868419&view=diff
==
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
Mon Oct 14 04:01:05 2019
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.run;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -218,7 +219,9 @@ public class DataStoreCommand implements
}
private static void logCliArgs(String[] args) {
-log.info("Command line arguments used for datastore command [{}]",
Joiner.on(' ').join(args));
+String [] filteredArgs = Arrays.stream(args).filter(
+str -> !str.startsWith("az:") &&
!str.startsWith("mongodb:")).toArray(String[]::new);
+log.info("Command line arguments used for datastore command [{}]",
Joiner.on(' ').join(filteredArgs));
List inputArgs =
ManagementFactory.getRuntimeMXBean().getInputArguments();
if (!inputArgs.isEmpty()) {
log.info("System properties and vm options passed {}", inputArgs);
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java?rev=1868419&r1=1868418&r2=1868419&view=diff
==
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Mon Oct 14 04:01:05 2019
@@ -87,7 +87,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -231,6 +230,38 @@ public class DataStoreCommandTest {
}
@Test
+public void testNoSensitiveParams() throws Exception {
+storeFixture.close();
+File dump = temporaryFolder.newFolder();
+List argsList = Lists
+.newArrayList("--check-consistency", "--" +
getOption(blobFixture.getType()), blobFixture.getConfigPath(), "--out-dir",
+dump.getAbsolutePath(), storeFixture.getConnectionString(),
"--reset-log-config", "false", "--work-dir",
+temporaryFolder.newFolder().getAbsolutePath());
+if (!Strings.isNullOrEmpty(additionalParams)) {
+argsList.add(additionalParams);
+}
+
+log.info("Running testNoSensitiveParams: {}", argsList);
+LogCustomizer customLogs = LogCustomizer
+.forLogger(DataStoreCommand.class.getName())
+.enable(Level.INFO)
+.filter(Level.INFO)
+.matchesRegex(".*mongodb:.*|.*az:.*")
+.create();
+customLogs.starting();
+
+DataStoreCommand cmd = new DataStoreCommand();
+try {
+cmd.execute(argsList.toArray(new String[0]));
+} catch (Exception e) {
+log.error("", e);
+}
+
+Assert.assertEquals(0, customLogs.getLogs().size());
+customLogs.finished();
+}
+
+@Test
public void testMissingOpParams() throws Exception {
storeFixture.close();
File dump = temporaryFolder.newFolder();
svn commit: r1867618 - /jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
Author: amitj
Date: Fri Sep 27 06:05:27 2019
New Revision: 1867618
URL: http://svn.apache.org/viewvc?rev=1867618&view=rev
Log:
OAK-8593: Enable a transient cluster-node to connect as invisible to oak
discovery
- Fixed test expectations
Modified:
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
Modified:
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java?rev=1867618&r1=1867617&r2=1867618&view=diff
==
---
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
(original)
+++
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
Fri Sep 27 06:05:27 2019
@@ -684,12 +684,14 @@ public abstract class BaseDocumentDiscov
logger.info("checkFiestaState: checking state. expected active:
"+activeIds+", inactive: "+inactiveIds);
for (Iterator it = instances.iterator();
it.hasNext();) {
SimplifiedInstance anInstance = it.next();
-
-final ViewExpectation e = new ViewExpectation(anInstance);
-e.setActiveIds(activeIds.toArray(new Integer[activeIds.size()]));
-e.setInactiveIds(inactiveIds.toArray(new
Integer[inactiveIds.size()]));
-waitFor(e, 6, "checkFiestaState failed for " + anInstance + ",
with instances: " + instances + ", and inactiveIds: "
+if (!anInstance.isInvisible()) {
+final ViewExpectation e = new ViewExpectation(anInstance);
+e.setActiveIds(activeIds.toArray(new
Integer[activeIds.size()]));
+e.setInactiveIds(inactiveIds.toArray(new
Integer[inactiveIds.size()]));
+waitFor(e, 6, "checkFiestaState failed for " + anInstance
+ ", with instances: " + instances + ","
++ " and inactiveIds: "
+ inactiveIds);
+}
}
}
@@ -845,9 +847,7 @@ public abstract class BaseDocumentDiscov
final SimplifiedInstance instance =
instances.remove(random.nextInt(instances.size()));
assertNotNull(instance.workingDir);
logger.info("Case 3: Shutdown instance: " +
instance.ns.getClusterId());
-if (!instance.isInvisible()) {
-inactiveIds.put(instance.ns.getClusterId(),
instance.workingDir);
-}
+inactiveIds.put(instance.ns.getClusterId(),
instance.workingDir);
instance.shutdown();
}
break;
svn commit: r1867571 - in /jackrabbit/oak/trunk: oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/ oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/ oak-store-document/sr
Author: amitj
Date: Thu Sep 26 10:48:14 2019
New Revision: 1867571
URL: http://svn.apache.org/viewvc?rev=1867571&view=rev
Log:
OAK-8593: Enable a transient cluster-node to connect as invisible to oak
discovery
Added an 'Invisible' flag in ClusterInfo
DocumentFixtureProvider in oak-run-commons now adds 'invisible' by default
Added:
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocumentTest.java
(with props)
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteInvisibleServiceCrashTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java
jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBroadcastConfig.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteService.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BaseDocumentDiscoveryLiteServiceTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterInfoTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoComparatorTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceIT.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentDiscoveryLiteServiceTest.java
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==
---
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureProvider.java
Thu Sep 26 10:48:14 2019
@@ -79,6 +79,7 @@ class DocumentFixtureProvider {
if (readOnly) {
builder.setReadOnlyMode();
}
+builder.setClusterInvisible(true);
int cacheSize = docStoreOpts.getCacheSize();
if (cacheSize != 0) {
Modified:
jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java?rev=1867571&r1=1867570&r2=1867571&view=diff
==
---
jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/run/cli/DocumentFixtureTest.java
Thu Sep 26 10:48:14 2019
@@ -20,8 +20,11 @@
package org.apache.jackrabbit.oak.run.cli;
import java.io.IOException;
+import java.util.List;
import joptsimple.OptionParser;
+import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfoDocument;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
import org.apache.jackrabbit.oak.plugins.document.MongoUtils;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -34,6 +37,7 @@ import org.junit.Test;
import static java.util.Collections.emptyMap;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -54,6 +58,7 @@ public class DocumentFixtureTest {
build
svn commit: r1866023 - in /jackrabbit/oak/trunk: oak-benchmarks/ oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/ oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/
Author: amitj
Date: Wed Aug 28 07:14:14 2019
New Revision: 1866023
URL: http://svn.apache.org/viewvc?rev=1866023&view=rev
Log:
OAK-8578: Introduce API to check whether blob inlined in Id
- Add a micro-benchmark for Binary#getUri
Added:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/GetURITest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-benchmarks/pom.xml
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
jackrabbit/oak/trunk/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java
Modified: jackrabbit/oak/trunk/oak-benchmarks/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/pom.xml?rev=1866023&r1=1866022&r2=1866023&view=diff
==
--- jackrabbit/oak/trunk/oak-benchmarks/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-benchmarks/pom.xml Wed Aug 28 07:14:14 2019
@@ -162,8 +162,14 @@
org.apache.jackrabbit
-jackrabbit-aws-ext
-${jackrabbit.version}
+oak-blob-cloud
+${project.version}
+true
+
+
+org.apache.jackrabbit
+oak-blob-cloud-azure
+${project.version}
true
Modified:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1866023&r1=1866022&r2=1866023&view=diff
==
---
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
(original)
+++
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
Wed Aug 28 07:14:14 2019
@@ -533,7 +533,8 @@ public class BenchmarkRunner {
new BasicWriteTest(),
new CanReadNonExisting(),
new IsNodeTypeTest(runAsAdmin.value(options)),
-new SetPropertyTransientTest()
+new SetPropertyTransientTest(),
+new GetURITest()
};
Set argset = Sets.newHashSet(nonOption.values(options));
Added:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/GetURITest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/GetURITest.java?rev=1866023&view=auto
==
---
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/GetURITest.java
(added)
+++
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/GetURITest.java
Wed Aug 28 07:14:14 2019
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+import java.net.URI;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Random;
+
+import javax.jcr.Binary;
+import javax.jcr.Node;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.jackrabbit.api.binary.BinaryDownload;
+import org.apache.jackrabbit.api.binary.BinaryDownloadOptions;
+
+public class GetURITest extends AbstractTest {
+
+private static final int FILE_COUNT = Integer.getInteger("file.count",
100);
+
+private static final int FILE_SIZE = Integer.getInteger("file.size", 20);
+
+private static final boolean DEBUG = Boolean.getBoolean("debug");
+
+private Session session;
+
+private Node root;
+
+private List binariesAdded = Lists.newArrayList();
+
+private Random random = new Random(29);
+
+/**
+ * Iteration counter used to avoid th
svn commit: r1866022 - in /jackrabbit/oak/trunk: oak-api/src/main/java/org/apache/jackrabbit/oak/api/ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/test/j
Author: amitj
Date: Wed Aug 28 07:13:59 2019
New Revision: 1866022
URL: http://svn.apache.org/viewvc?rev=1866022&view=rev
Log:
OAK-8578: Introduce API to check whether blob inlined in Id
OAK-8552: Minimize network calls required when creating a direct download URI
Add a new method #isInlined to Blob class to check if the external blob is id
inlined
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BlobIdRecordTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/value/jcr/BinaryImpl.java
jackrabbit/oak/trunk/oak-store-spi/src/test/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlobTest.java
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java?rev=1866022&r1=1866021&r2=1866022&view=diff
==
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
Wed Aug 28 07:13:59 2019
@@ -87,4 +87,12 @@ public interface Blob {
*/
@Nullable
String getContentIdentity();
+
+/**
+ * Checks if the external blob is inlined in the id itself
+ * @return
+ */
+default boolean isInlined() {
+return false;
+}
}
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java?rev=1866022&r1=1866021&r2=1866022&view=diff
==
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
Wed Aug 28 07:13:59 2019
@@ -18,7 +18,7 @@
/**
* Oak repository API
*/
-@Version("3.2.0")
+@Version("3.3.0")
package org.apache.jackrabbit.oak.api;
import org.osgi.annotation.versioning.Version;
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java?rev=1866022&r1=1866021&r2=1866022&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
Wed Aug 28 07:13:59 2019
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.io.InputStream;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.api.Blob;
import org.jetbrains.annotations.NotNull;
@@ -79,6 +80,11 @@ public class BlobStoreBlob implements Bl
return blobId;
}
+@Override
+public boolean isInlined() {
+return InMemoryDataRecord.isInstance(blobId);
+}
+
public String getBlobId() {
return blobId;
}
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java?rev=1866022&r1=1866021&r2=1866022&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blo
svn commit: r1865974 - in /jackrabbit/oak/trunk: oak-api/src/main/java/org/apache/jackrabbit/oak/api/ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/test/j
Author: amitj
Date: Tue Aug 27 08:06:58 2019
New Revision: 1865974
URL: http://svn.apache.org/viewvc?rev=1865974&view=rev
Log:
OAK-8578:Reverting commit r1865962
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/proc/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BigInlinedBinaryIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BlobIdRecordTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/value/BinaryBasedBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/value/jcr/BinaryImpl.java
jackrabbit/oak/trunk/oak-store-spi/src/test/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlobTest.java
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java?rev=1865974&r1=1865973&r2=1865974&view=diff
==
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
Tue Aug 27 08:06:58 2019
@@ -87,10 +87,4 @@ public interface Blob {
*/
@Nullable
String getContentIdentity();
-
-/**
- * Checks if the external blob is inlined in the id itself
- * @return
- */
-boolean isInlined();
}
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java?rev=1865974&r1=1865973&r2=1865974&view=diff
==
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
Tue Aug 27 08:06:58 2019
@@ -18,7 +18,7 @@
/**
* Oak repository API
*/
-@Version("4.0.0")
+@Version("3.2.0")
package org.apache.jackrabbit.oak.api;
import org.osgi.annotation.versioning.Version;
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java?rev=1865974&r1=1865973&r2=1865974&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
Tue Aug 27 08:06:58 2019
@@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.io.InputStream;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.api.Blob;
import org.jetbrains.annotations.NotNull;
@@ -80,10 +79,6 @@ public class BlobStoreBlob implements Bl
return blobId;
}
-@Override public boolean isInlined() {
-return InMemoryDataRecord.isInstance(blobId);
-}
-
public String getBlobId() {
return blobId;
}
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBl
svn propchange: r1865962 - svn:log
Author: amitj Revision: 1865962 Modified property: svn:log Modified: svn:log at Tue Aug 27 05:10:40 2019 -- --- svn:log (original) +++ svn:log Tue Aug 27 05:10:40 2019 @@ -1,3 +1,5 @@ +OAK-8578: Introduce API to check whether blob inlined in Id OAK-8552: Minimize network calls required when creating a direct download URI Add a new method #isInlined to Blob class to check if the external blob is id inlined +Using the new API instead of getReference to check if blob inlined in BinaryImpl#getURI
svn commit: r1865962 - in /jackrabbit/oak/trunk: oak-api/src/main/java/org/apache/jackrabbit/oak/api/ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/test/j
Author: amitj
Date: Tue Aug 27 04:08:42 2019
New Revision: 1865962
URL: http://svn.apache.org/viewvc?rev=1865962&view=rev
Log:
OAK-8552: Minimize network calls required when creating a direct download URI
Add a new method #isInlined to Blob class to check if the external blob is id
inlined
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/proc/SegmentBlob.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BigInlinedBinaryIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BlobIdRecordTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ExternalBlobIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/value/BinaryBasedBlob.java
jackrabbit/oak/trunk/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/value/jcr/BinaryImpl.java
jackrabbit/oak/trunk/oak-store-spi/src/test/java/org/apache/jackrabbit/oak/plugins/memory/AbstractBlobTest.java
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java?rev=1865962&r1=1865961&r2=1865962&view=diff
==
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/Blob.java
Tue Aug 27 04:08:42 2019
@@ -87,4 +87,10 @@ public interface Blob {
*/
@Nullable
String getContentIdentity();
+
+/**
+ * Checks if the external blob is inlined in the id itself
+ * @return
+ */
+boolean isInlined();
}
Modified:
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java?rev=1865962&r1=1865961&r2=1865962&view=diff
==
---
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
(original)
+++
jackrabbit/oak/trunk/oak-api/src/main/java/org/apache/jackrabbit/oak/api/package-info.java
Tue Aug 27 04:08:42 2019
@@ -18,7 +18,7 @@
/**
* Oak repository API
*/
-@Version("3.2.0")
+@Version("4.0.0")
package org.apache.jackrabbit.oak.api;
import org.osgi.annotation.versioning.Version;
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java?rev=1865962&r1=1865961&r2=1865962&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobStoreBlob.java
Tue Aug 27 04:08:42 2019
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.io.InputStream;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.api.Blob;
import org.jetbrains.annotations.NotNull;
@@ -79,6 +80,10 @@ public class BlobStoreBlob implements Bl
return blobId;
}
+@Override public boolean isInlined() {
+return InMemoryDataRecord.isInstance(blobId);
+}
+
public String getBlobId() {
return blobId;
}
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL:
http://svn.apac
svn commit: r1862717 - in /jackrabbit/oak/branches/1.10: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/pl
Author: amitj
Date: Mon Jul 8 06:55:33 2019
New Revision: 1862717
URL: http://svn.apache.org/viewvc?rev=1862717&view=rev
Log:
OAK-8314: BlobIds created with direct uploads lack length suffix
OAK-8394: Fix BinaryAccessDSGCIT failing intermittently for Mongo
Merge r1859350, r1859351, r1859365, r1860931 from trunk
Added:
jackrabbit/oak/branches/1.10/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
- copied, changed from r1859351,
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
jackrabbit/oak/branches/1.10/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
- copied unchanged from r1859351,
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
jackrabbit/oak/branches/1.10/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
jackrabbit/oak/branches/1.10/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
Propchange: jackrabbit/oak/branches/1.10/
--
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jul 8 06:55:33 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852135,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853083,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854055,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854539,1854701,1854773-1854774,1854827,1854848,1854859,1854930,1854990-1854991,1855032,1855221,1855477-1855478,1855776,1855993,1856049,1856056,1856538,1856545,1857000,1857010,1857104,1857159,1857212,1857221,1857238,1857247,1857253,1857294,1857314,1857463,1857480,1857577,1857589,1857592,1857627,1857634-1857635,1857638,1857640,1857687,1857936,1858032,1858053,1858123,1858139,1858385,1858424,1858571,1858578,1858810,1858926,1858931,1859020,1859231,1859292,1859294,1859359,1859533,1859609,1859612,1859619,1859711,1859716,1859772,1859776,1859780,1859843,1859854,1859881
,1860120,1860131,1860137,1860202,1860278,1860328,1860330,1860355,1860393,1860442,1860548,1860564-1860565,1861270,1861626,1862044,1862093,1862531
+/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852135,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853083,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854055,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854539,1854701,1854773-1854774,1854827,1854848,1854859,1854930,1854990-1854991,1855032,1855221,1855477-1855478,1855776,1855993,1856049,1856056,1856538,1856545,1857000,1857010,1857104,1857159,1857212,1857221,1857238,1857247,1857253,1857294,1857314,1857463,1857480,1857577,1857589,1857592,1857627,1857634-1857635,1857638,1857640,1857687,1857936,1858032,1858053,1858123,1858139,1858385,1858424,1858571,1858578,1858810,1858926,1858931,1859020,1859231,1859292,1859294,1859350-1859351,1859359,1859365,1859533,1859609,1859612,1859619,1859711,1859716,1859772,1859776,1859780
,1859843,1859854,1859881,1860120,1860131,1860137,1860202,1860278,1860328,1860330,1860355,1860393,1860442,1860548,1860564-1860565,1860931,1861270,1861626,1862044,1862093,1862531
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1862717&r1=1862716&r2=1862717&view=diff
==
---
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Mon Jul 8 06:55:33 2019
@@ -724,7 +724,7 @@ public class DataStoreBlobStore
if (delegate instanceof DataRecordAccessProvider) {
svn commit: r1862713 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java test/java/org/apache/jackrabbit/oak/plugins/blob/B
Author: amitj
Date: Mon Jul 8 04:15:13 2019
New Revision: 1862713
URL: http://svn.apache.org/viewvc?rev=1862713&view=rev
Log:
OAK-8463: Fix merge in consistency check to allow paths suffixed to blob ids
- Fix merge to ignore paths if present
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1862713&r1=1862712&r2=1862713&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jul 8 04:15:13 2019
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Lists.newArrayList;
import static java.io.File.createTempFile;
+import static java.util.Comparator.comparing;
import static org.apache.commons.io.FileUtils.copyFile;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.copy;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.merge;
@@ -702,7 +703,7 @@ public class MarkSweepGarbageCollector i
File file = copy(refFile.getStream());
files.add(file);
}
-merge(files, fs.getMarkedRefs());
+merge(files, fs.getMarkedRefs(), comparing(s ->
s.split(DELIM)[0]));
}
}
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1862713&r1=1862712&r2=1862713&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Mon Jul 8 04:15:13 2019
@@ -94,6 +94,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.mockito.internal.util.collections.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -386,10 +387,11 @@ public class BlobGCTest {
log.info("Staring checkConsistencyGlobal()");
// Setup a different cluster/repository sharing the blob store
-MemoryBlobStoreNodeStore secondClusterNodeStore = new
MemoryBlobStoreNodeStore(cluster.blobStore);
+MemoryBlobStoreNodeStore secondClusterNodeStore = new
MemoryBlobStoreNodeStore(cluster.blobStore, true);
Cluster secondCluster = new Cluster(folder.newFolder(),
cluster.blobStore, secondClusterNodeStore, 100);
closer.register(secondCluster);
+
secondCluster.blobStoreState.blobsPresent.add(Iterables.firstOf(cluster.blobStoreState.blobsPresent));
// Execute mark on the default cluster
executeGarbageCollection(cluster, cluster.getCollector(0), true);
MarkSweepGarbageCollector globalCollector =
secondCluster.getCollector(0, true);
@@ -642,10 +644,16 @@ public class BlobGCTest {
*/
public static class MemoryBlobStoreNodeStore extends MemoryNodeStore {
private final BlobStore blobStore;
+private final boolean fakePath;
Set referencedBlobs;
public MemoryBlobStoreNodeStore(BlobStore blobStore) {
+this(blobStore, false);
+}
+
+public MemoryBlobStoreNodeStore(BlobStore blobStore, boolean fakePath)
{
this.blobStore = blobStore;
+this.fakePath = fakePath;
}
public void setReferencedBlobs(Set referencedBlobs) {
@@ -669,8 +677,8 @@ public class BlobGCTest {
public BlobReferenceRetriever getBlobReferenceRetriever() {
return collector -> {
-for(String id : referencedBlobs) {
-collector.addReference(id, null);
+for (String id : referencedBlobs) {
+collector.addReference(id, (fakePath ?
UUID.randomUUID().toString() : null));
}
};
}
svn propchange: r1862378 - svn:log
Author: amitj Revision: 1862378 Modified property: svn:log Modified: svn:log at Mon Jul 1 14:12:28 2019 -- --- svn:log (original) +++ svn:log Mon Jul 1 14:12:28 2019 @@ -1,3 +1,3 @@ -OAK-8446: Enable datastore consistency check after completion of garbage collection +OAK-8447: Handle all available references in the DataStore when doing consistency check skip handling specific to SharedDataStore when the BlobStore is not a SharedDataStore
svn propchange: r1862380 - svn:log
Author: amitj Revision: 1862380 Modified property: svn:log Modified: svn:log at Mon Jul 1 14:10:55 2019 -- --- svn:log (original) +++ svn:log Mon Jul 1 14:10:55 2019 @@ -1,3 +1,3 @@ -OAK-8446: Enable datastore consistency check after completion of garbage collection +OAK-8447: Handle all available references in the DataStore when doing consistency check close (another) FileReader when done
svn commit: r1862381 - /jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Author: amitj
Date: Mon Jul 1 14:02:51 2019
New Revision: 1862381
URL: http://svn.apache.org/viewvc?rev=1862381&view=rev
Log:
OAK-8447: Handle all available references in the DataStore when doing
consistency check
- Fixed check for SharedDataStore
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1862381&r1=1862380&r2=1862381&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jul 1 14:02:51 2019
@@ -687,10 +687,10 @@ public class MarkSweepGarbageCollector i
throw e;
}
-// Retrieve all other marked present in the datastore
-if (blobStore instanceof SharedDataStore) {
-List refFiles = ((SharedDataStore) blobStore)
-
.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+if (SharedDataStoreUtils.isShared(blobStore)) {
+// Retrieve all other marked present in the datastore
+List refFiles =
+((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
if (refFiles.size() > 0) {
File temp = new File(root, repoId +
UUID.randomUUID().toString());
copyFile(fs.getMarkedRefs(), temp);
svn commit: r1862366 - in /jackrabbit/oak/trunk: oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ oak-run/s
Author: amitj
Date: Mon Jul 1 10:22:03 2019
New Revision: 1862366
URL: http://svn.apache.org/viewvc?rev=1862366&view=rev
Log:
OAK-8446: Enable datastore consistency check after completion of garbage
collection
- Enables datastore consistency in a low cost way after GC
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1862366&r1=1862365&r2=1862366&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jul 1 10:22:03 2019
@@ -23,12 +23,14 @@ import static org.apache.commons.io.File
import static org.apache.jackrabbit.oak.commons.FileIOUtils.copy;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.merge;
import static org.apache.jackrabbit.oak.commons.FileIOUtils.sort;
+import static org.apache.jackrabbit.oak.commons.FileIOUtils.writeStrings;
import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
@@ -116,6 +118,9 @@ public class MarkSweepGarbageCollector i
/** The blob store to be garbage collected. */
private final GarbageCollectableBlobStore blobStore;
+/** Flag to enable low cost consistency check after DSGC */
+private boolean checkConsistencyAfterGc;
+
/** Helper class to mark blob references which **/
private final BlobReferenceRetriever marker;
@@ -167,12 +172,14 @@ public class MarkSweepGarbageCollector i
String root,
int batchCount,
long maxLastModifiedInterval,
+boolean checkConsistencyAfterGc,
@Nullable String repositoryId,
@Nullable Whiteboard whiteboard,
@Nullable StatisticsProvider statisticsProvider)
throws IOException {
this.executor = executor;
this.blobStore = blobStore;
+this.checkConsistencyAfterGc = checkConsistencyAfterGc;
checkNotNull(blobStore, "BlobStore cannot be null");
this.marker = marker;
this.batchCount = batchCount;
@@ -204,7 +211,7 @@ public class MarkSweepGarbageCollector i
long maxLastModifiedInterval,
@Nullable String repositoryId)
throws IOException {
-this(marker, blobStore, executor, root, batchCount,
maxLastModifiedInterval, repositoryId, null, null);
+this(marker, blobStore, executor, root, batchCount,
maxLastModifiedInterval, false, repositoryId, null, null);
}
/**
@@ -219,7 +226,7 @@ public class MarkSweepGarbageCollector i
@Nullable Whiteboard whiteboard,
@Nullable StatisticsProvider statisticsProvider)
throws IOException {
-this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT,
maxLastModifiedInterval, repositoryId, whiteboard, statisticsProvider);
+this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT,
maxLastModifiedInterval, false, repositoryId, whiteboard, statisticsProvider);
}
@Override
@@ -506,6 +513,9 @@ public class MarkSweepGarbageCollector i
closeQuietly(removesWriter);
}
+if (checkConsistencyAfterGc) {
+
BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs,
consistencyStatsCollector, new File(root));
+}
BlobCollectionType.get(blobStore).handleRemoves(blobStore,
fs.getGarbage(), fs.getMarkedRefs());
if(count != deleted) {
@@ -1006,6 +1016,50 @@ public class MarkSweepGarbageCollector i
FileUtils.forceDelete(removedIds);
}
+void checkConsistencyAfterGC(GarbageColl
svn commit: r1862367 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java test/java/org/apache/jackrabbit/oak/plugins/blob/B
Author: amitj
Date: Mon Jul 1 10:22:19 2019
New Revision: 1862367
URL: http://svn.apache.org/viewvc?rev=1862367&view=rev
Log:
OAK-8447: Handle all available references in the DataStore when doing
consistency check
- Now if available all reference files will be accounted for when doing
consistency check
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1862367&r1=1862366&r2=1862367&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jul 1 10:22:19 2019
@@ -687,6 +687,23 @@ public class MarkSweepGarbageCollector i
throw e;
}
+// Retrieve all other marked present in the datastore
+List refFiles =
+((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+if (refFiles.size() > 0) {
+File temp = new File(root, repoId +
UUID.randomUUID().toString());
+copyFile(fs.getMarkedRefs(), temp);
+
+// List of files to be merged
+List files = newArrayList();
+files.add(temp);
+for (DataRecord refFile : refFiles) {
+File file = copy(refFile.getStream());
+files.add(file);
+}
+merge(files, fs.getMarkedRefs());
+}
+
LOG.trace("Starting difference phase of the consistency check");
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getAvailableRefs(),
@@ -695,10 +712,16 @@ public class MarkSweepGarbageCollector i
// If tracking then also filter ids being tracked which are active
deletions for lucene
candidates = BlobCollectionType.get(blobStore).filter(blobStore,
iter, fs);
+
GarbageCollectionType.get(blobStore).removeAllMarkedReferences(blobStore);
+
LOG.trace("Ending difference phase of the consistency check");
LOG.info("Consistency check found [{}] missing blobs", candidates);
if (candidates > 0) {
+LineIterator lineIterator = new LineIterator(new
FileReader(fs.getGcCandidates()));
+while(lineIterator.hasNext()) {
+LOG.warn("Missing Blob [{}]", lineIterator.nextLine());
+}
LOG.warn("Consistency check failure in the the blob store :
{}, check missing candidates in file {}",
blobStore, fs.getGcCandidates().getAbsolutePath());
consistencyStatsCollector.finishFailure();
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1862367&r1=1862366&r2=1862367&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Mon Jul 1 10:22:19 2019
@@ -382,6 +382,68 @@ public class BlobGCTest {
}
@Test
+public void checkConsistencyGlobal() throws Exception {
+log.info("Staring checkConsistencyGlobal()");
+
+// Setup a different cluster/repository sharing the blob store
+MemoryBlobStoreNodeStore secondClusterNodeStore = new
MemoryBlobStoreNodeStore(cluster.blobStore);
+Cluster secondCluster = new Cluster(folder.newFolder(),
cluster.blobStore, secondClusterNodeStore, 100);
+closer.register(secondCluster);
+
+// Execute mark on the default cluster
+executeGarbageCollection(cluster, cluster.getCollector(0), true);
+MarkSweepGarbageCollector globalCollector =
secondCluster.getCollector(0, true);
+long missing = globalCollector.checkConsistency();
+assertEquals(0, missing);
+as
svn commit: r1860931 - /jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Author: amitj
Date: Mon Jun 10 08:44:23 2019
New Revision: 1860931
URL: http://svn.apache.org/viewvc?rev=1860931&view=rev
Log:
OAK-8394: Fix BinaryAccessDSGCIT failing intermittently for Mongo
Catching exception if any to ignore tests and also moved init of mongo
connection factory only when datastore available.
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java?rev=1860931&r1=1860930&r2=1860931&view=diff
==
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Mon Jun 10 08:44:23 2019
@@ -64,20 +64,33 @@ public class DocumentMongoNodeStoreFixtu
private final Table components =
HashBasedTable.create();
private MongoConnection connection;
private final Clock clock;
-public final MongoConnectionFactory connFactory = new
MongoConnectionFactory();
+public MongoConnectionFactory connFactory;
private String db;
public DocumentMongoNodeStoreFixture(@Nullable DataStoreFixture
dataStoreFixture) {
this.dataStoreFixture = dataStoreFixture;
this.clock = new Clock.Virtual();
}
+/**
+ * Mandatory to be called to initialize the connectionFactory.
+ * Lazy initializes it to limit docker container init only if relevant
datastores available.
+ *
+ * @return
+ */
@Override
public boolean isAvailable() {
db = UUID.randomUUID().toString();
-this.connection = connFactory.getConnection(db);
// if a DataStore is configured, it must be available for our
NodeStore to be available
-return (dataStoreFixture == null || dataStoreFixture.isAvailable()) &&
(connection != null);
+if ((dataStoreFixture == null || dataStoreFixture.isAvailable())) {
+try {
+this.connFactory = new MongoConnectionFactory();
+this.connection = connFactory.getConnection(db);
+
+return (connection != null);
+} catch (Exception e) {}
+}
+return false;
}
@Override
@@ -126,7 +139,7 @@ public class DocumentMongoNodeStoreFixtu
@Override
public void dispose(NodeStore nodeStore) {
try {
-if (nodeStore instanceof DocumentNodeStore) {
+if (nodeStore != null && nodeStore instanceof DocumentNodeStore) {
((DocumentNodeStore)nodeStore).dispose();
}
@@ -138,7 +151,9 @@ public class DocumentMongoNodeStoreFixtu
FileUtils.deleteQuietly(dataStoreFolder);
}
MongoUtils.dropDatabase(db);
-connection.close();
+if (connection != null) {
+connection.close();
+}
} finally {
components.row(nodeStore).clear();
}
svn commit: r1859365 - /jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Author: amitj
Date: Thu May 16 10:08:56 2019
New Revision: 1859365
URL: http://svn.apache.org/viewvc?rev=1859365&view=rev
Log:
OAK-8314: BlobIds created with direct uploads lack length suffix
Proper DB cleanup for DocumentNodeStore-Mongo fixture for BinaryAccessDSGCIT
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java?rev=1859365&r1=1859364&r2=1859365&view=diff
==
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Thu May 16 10:08:56 2019
@@ -20,6 +20,7 @@ package org.apache.jackrabbit.oak.jcr.bi
import java.io.File;
import java.io.IOException;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.jcr.RepositoryException;
@@ -37,6 +38,7 @@ import org.apache.jackrabbit.oak.plugins
import
org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory;
+import org.apache.jackrabbit.oak.plugins.document.MongoUtils;
import org.apache.jackrabbit.oak.plugins.document.Revision;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
import
org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentNodeStoreBuilder;
@@ -63,7 +65,7 @@ public class DocumentMongoNodeStoreFixtu
private MongoConnection connection;
private final Clock clock;
public final MongoConnectionFactory connFactory = new
MongoConnectionFactory();
-
+private String db;
public DocumentMongoNodeStoreFixture(@Nullable DataStoreFixture
dataStoreFixture) {
this.dataStoreFixture = dataStoreFixture;
this.clock = new Clock.Virtual();
@@ -71,7 +73,8 @@ public class DocumentMongoNodeStoreFixtu
@Override
public boolean isAvailable() {
-this.connection = connFactory.getConnection();
+db = UUID.randomUUID().toString();
+this.connection = connFactory.getConnection(db);
// if a DataStore is configured, it must be available for our
NodeStore to be available
return (dataStoreFixture == null || dataStoreFixture.isAvailable()) &&
(connection != null);
@@ -134,6 +137,7 @@ public class DocumentMongoNodeStoreFixtu
File dataStoreFolder = (File) components.get(nodeStore,
DataStore.class.getName() + ":folder");
FileUtils.deleteQuietly(dataStoreFolder);
}
+MongoUtils.dropDatabase(db);
connection.close();
} finally {
components.row(nodeStore).clear();
svn commit: r1859351 - in /jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary: ./ fixtures/nodestore/ util/
Author: amitj
Date: Thu May 16 06:52:07 2019
New Revision: 1859351
URL: http://svn.apache.org/viewvc?rev=1859351&view=rev
Log:
OAK-8314: BlobIds created with direct uploads lack length suffix
Enable DocumentNodeStore-Mongo fixture for BinaryAccessDSGCIT
Added:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
- copied, changed from r1859350,
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java?rev=1859351&r1=1859350&r2=1859351&view=diff
==
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
Thu May 16 06:52:07 2019
@@ -56,25 +56,33 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.datastore.AzureDataStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.datastore.S3DataStoreFixture;
+import
org.apache.jackrabbit.oak.jcr.binary.fixtures.nodestore.DocumentMongoNodeStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.nodestore.SegmentMemoryNodeStoreFixture;
+import org.apache.jackrabbit.oak.jcr.binary.util.BinaryAccessDSGCFixture;
import org.apache.jackrabbit.oak.jcr.binary.util.Content;
+import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
-import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runners.Parameterized;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BinaryAccessDSGCIT extends AbstractBinaryAccessIT {
+private static Logger LOG =
LoggerFactory.getLogger(BinaryAccessDSGCIT.class);
+
private static final String TEST_ROOT = "testroot";
private static final long BINARY_SIZE = 1024*1024;
@@ -83,17 +91,24 @@ public class BinaryAccessDSGCIT extends
private static final String DIRECT_UPLOAD_1 = "du1";
private static final String DIRECT_UPLOAD_2 = "du2";
+@Rule
+public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
@Parameterized.Parameters(name = "{0}")
public static Iterable dataStoreFixtures() {
Collection fixtures = new ArrayList<>();
fixtures.add(new SegmentMemoryNodeStoreFixture(new
S3DataStoreFixture()));
+fixtures.add(new DocumentMongoNodeStoreFixture(new
S3DataStoreFixture()));
+
fixtures.add(new SegmentMemoryNodeStoreFixture(new
AzureDataStoreFixture()));
+fixtures.add(new DocumentMongoNodeStoreFixture(new
AzureDataStoreFixture()));
+
return fixtures;
}
public BinaryAccessDSGCIT(NodeStoreFixture fixture) {
// reuse NodeStore (and DataStore) across all tests in this class
-super(fixture, true);
+super(fixture, false);
}
private Session session;
@@ -190,8 +205,6 @@ public class BinaryAccessDSGCIT extends
private MarkSweepGarbageCollector getGarbageCollector()
throws DataStoreException, IOException {
DataStoreBlobStore blobStore = (DataStoreBlobStore)
getNodeStoreComponent(BlobStore.class);
-FileStore fileStore = getNodeStoreComponent(FileStore.class);
-File fileStoreRoot = getNodeStoreComponent(FileStore.class.getName() +
":r
svn commit: r1859350 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java test/java/org/apache/jackrabbit/oak/plugins/blo
Author: amitj
Date: Thu May 16 06:51:55 2019
New Revision: 1859350
URL: http://svn.apache.org/viewvc?rev=1859350&view=rev
Log:
OAK-8314: BlobIds created with direct uploads lack length suffix
Suffixing length to the ids being returned for for directly uploded binaries
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1859350&r1=1859349&r2=1859350&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Thu May 16 06:51:55 2019
@@ -724,7 +724,7 @@ public class DataStoreBlobStore
if (delegate instanceof DataRecordAccessProvider) {
try {
DataRecord record = ((DataRecordAccessProvider)
delegate).completeDataRecordUpload(uploadToken);
-return new BlobStoreBlob(this,
record.getIdentifier().toString());
+return new BlobStoreBlob(this, getBlobId(record));
}
catch (DataStoreException | DataRecordUploadException e) {
log.warn("Unable to complete direct upload for upload token
{}", uploadToken, e);
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1859350&r1=1859349&r2=1859350&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Thu May 16 06:51:55 2019
@@ -26,19 +26,25 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.net.URI;
import java.security.DigestOutputStream;
import java.security.MessageDigest;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicReference;
+import javax.jcr.RepositoryException;
+
import ch.qos.logback.classic.Level;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
@@ -49,15 +55,25 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.core.data.MultiDataStoreAware;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.api.blob.BlobAccessProvider;
+import org.apache.jackrabbit.oak.api.blob.BlobUpload;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordAccessProvider;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
+import
org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
-import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
import org.apache.jack
svn commit: r1855842 - in /jackrabbit/oak/trunk/oak-run: ./ src/main/assembly/ src/main/java/org/apache/jackrabbit/oak/run/ src/test/java/org/apache/jackrabbit/oak/run/
Author: amitj Date: Tue Mar 19 15:25:36 2019 New Revision: 1855842 URL: http://svn.apache.org/viewvc?rev=1855842&view=rev Log: OAK-8113: Add ability to push metrics to prometheus pushgateway from oak-run - Added an option --export-metrics to enable export - MetricsExporterFixture and FixtureProvider initialize metrics exporter based on type (currently only pushgateway). - Additional io.prometheus.simpleclient* libraries made optional and not part of the jar and would need to be specified externally on the classpath Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixture.java (with props) jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProvider.java (with props) jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/run/MetricsExporterFixtureProviderTest.java (with props) Modified: jackrabbit/oak/trunk/oak-run/README.md jackrabbit/oak/trunk/oak-run/pom.xml jackrabbit/oak/trunk/oak-run/src/main/assembly/oak-run.xml jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java Modified: jackrabbit/oak/trunk/oak-run/README.md URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1855842&r1=1855841&r2=1855842&view=diff == --- jackrabbit/oak/trunk/oak-run/README.md (original) +++ jackrabbit/oak/trunk/oak-run/README.md Tue Mar 19 15:25:36 2019 @@ -517,6 +517,7 @@ Maintenance commands for the DataStore: [--max-age ] \ [--verbose] \ [|] +[--metrics] [--export-metrics] The following operations are available: @@ -539,9 +540,32 @@ The following options are available: - Path to the tar segment store or the segment azure uri as specified in http://jackrabbit.apache.org/oak/docs/nodestore/segment/overview.html#remote-segment-stores or if Mongo NodeStore then the mongo uri. +--metrics- If metrics are to be captured. +--export-metrics - Option to export the captured metrics. The format of the command is type;URL;key1=value1,key2=value2 + Currently only [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) is supported + e.g. --export-metrics "pushgateway;localhost:9091;key1=value1,key2=value2" Note: +Note: When using --export-metrics the following additional jars have to be downloaded to support Prometheus Pushgatway +* [simpleclient_common-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient_common/0.6.0/simpleclient_common-0.6.0.jar) +* [simpleclient-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient/0.6.0/simpleclient-0.6.0.jar) +* [simpleclient_pushgateway-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient_pushgateway/0.6.0/simpleclient_pushgateway-0.6.0.jar) +* [simpleclient_dropwizard-0.6.0.jar](http://central.maven.org/maven2/io/prometheus/simpleclient_dropwizard/0.6.0/simpleclient_dropwizard-0.6.0.jar) + +The command to be executed when using this option is: + +$ java -classpath oak-run-*.jar:simpleclient_common-0.6.0.jar:simpleclient-0.6.0.jar:simpleclient_dropwizard-0.6.0.jar:simpleclient_pushgateway-0.6.0.jar \ +org.apache.jackrabbit.oak.run.Main \ +datastore [--check-consistency|--collect-garbage [true]] \ +[--s3ds |--fds |--azureds |fake-ds-path ] \ +[--out-dir ] \ +[--work-dir ] \ +[--max-age ] \ +[--verbose] \ +[|] +[--metrics] [--export-metrics] + Data Store and node store configuration is mandatory. The config files should be formatted according to the OSGi configuration admin specification Modified: jackrabbit/oak/trunk/oak-run/pom.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/pom.xml?rev=1855842&r1=1855841&r2=1855842&view=diff == --- jackrabbit/oak/trunk/oak-run/pom.xml (original) +++ jackrabbit/oak/trunk/oak-run/pom.xml Tue Mar 19 15:25:36 2019 @@ -372,7 +372,28 @@ org.jetbrains annotations - + + + + io.prometheus + simpleclient + 0.6.0 + true + + + io.prometheus + simpleclient_dropwizard + 0.6.0 + true + + + + io.prometheus + simpleclient_pushgateway + 0.6.0 + true + + junit Modified: jackrabbit/oak/trunk/oak-run/src/main/assembly/oak-run.xml URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/assembly/oak-run.xml?rev=1
svn commit: r1855603 - in /jackrabbit/oak/branches/1.10: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/pl
Author: amitj
Date: Fri Mar 15 14:24:29 2019
New Revision: 1855603
URL: http://svn.apache.org/viewvc?rev=1855603&view=rev
Log:
OAK-8115: Allow disabling of BlobTracker when not required
- Merge r1855478 from trunk
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
jackrabbit/oak/branches/1.10/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
jackrabbit/oak/branches/1.10/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Propchange: jackrabbit/oak/branches/1.10/
--
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Mar 15 14:24:29 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854701,1854827,1854848,1854859,1854930,1854990-1854991,1855221,1855477
+/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854701,1854827,1854848,1854859,1854930,1854990-1854991,1855221,1855477-1855478
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==
---
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
Fri Mar 15 14:24:29 2019
@@ -23,7 +23,6 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
@@ -53,6 +52,7 @@ import static com.google.common.io.Files
import static com.google.common.io.Files.newWriter;
import static java.io.File.createTempFile;
import static java.lang.System.currentTimeMillis;
+import static java.util.Collections.emptyIterator;
import static java.util.Collections.synchronizedList;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
@@ -83,41 +83,35 @@ import static org.apache.jackrabbit.oak.
public class BlobIdTracker implements Closeable, BlobTracker {
private static final Logger LOG =
LoggerFactory.getLogger(BlobIdTracker.class);
-/**
- * System property to skip tracker. If set will skip:
- * * Snapshots (No-op)
- * * Retrieve (return empty)
- * * Add (No-op)
- */
-private final boolean SKIP_TRACKER =
Boolean.getBoolean("oak.datastore.skipTracker");
-
private static final String datastoreMeta = "blobids";
private static final String fileNamePrefix = "blob";
private static final String mergedFileSuffix = ".refs";
private static final String snapshotMarkerSuffix = ".snapshot";
/* Local instance identifier */
-private final String instanceId = randomUUID().toString();
+private String instanceId = rando
svn commit: r1855574 - in /jackrabbit/oak/branches/1.10: ./ oak-run/src/main/resources/logback-datastore.xml
Author: amitj
Date: Fri Mar 15 08:37:19 2019
New Revision: 1855574
URL: http://svn.apache.org/viewvc?rev=1855574&view=rev
Log:
OAK-8112: Change log pattern to have complete date for oak-run datastore command
- Merge r1855477 from trunk
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-run/src/main/resources/logback-datastore.xml
Propchange: jackrabbit/oak/branches/1.10/
--
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Mar 15 08:37:19 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854701,1854827,1854848,1854859,1854930,1854990-1854991,1855221
+/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854701,1854827,1854848,1854859,1854930,1854990-1854991,1855221,1855477
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.10/oak-run/src/main/resources/logback-datastore.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-run/src/main/resources/logback-datastore.xml?rev=1855574&r1=1855573&r2=1855574&view=diff
==
---
jackrabbit/oak/branches/1.10/oak-run/src/main/resources/logback-datastore.xml
(original)
+++
jackrabbit/oak/branches/1.10/oak-run/src/main/resources/logback-datastore.xml
Fri Mar 15 08:37:19 2019
@@ -22,14 +22,14 @@
${oak.workDir}/datastore.log
- %d %-5level [%thread] %logger{30} %marker- %msg %n
+ %d{dd.MM. HH:mm:ss.SSS} %-5(*%level*) [%thread] %logger{30}
%marker- %msg %n
System.out
- %d{HH:mm:ss} - %msg%n
+ %d{dd.MM. HH:mm:ss.SSS} %-5(*%level*) [%thread] %logger{30}
%marker- %msg %n
svn commit: r1855478 - in /jackrabbit/oak/trunk: oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/
Author: amitj
Date: Thu Mar 14 03:17:01 2019
New Revision: 1855478
URL: http://svn.apache.org/viewvc?rev=1855478&view=rev
Log:
OAK-8115: Allow disabling of BlobTracker when not required
- BlobTracker can be disabled by setting config property snapshotIntervalSecs
to 0 or below as this prop needs a sensible non-zero value anyways to be
effective
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
jackrabbit/oak/trunk/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1855478&r1=1855477&r2=1855478&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
Thu Mar 14 03:17:01 2019
@@ -23,7 +23,6 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
@@ -53,6 +52,7 @@ import static com.google.common.io.Files
import static com.google.common.io.Files.newWriter;
import static java.io.File.createTempFile;
import static java.lang.System.currentTimeMillis;
+import static java.util.Collections.emptyIterator;
import static java.util.Collections.synchronizedList;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
@@ -83,41 +83,35 @@ import static org.apache.jackrabbit.oak.
public class BlobIdTracker implements Closeable, BlobTracker {
private static final Logger LOG =
LoggerFactory.getLogger(BlobIdTracker.class);
-/**
- * System property to skip tracker. If set will skip:
- * * Snapshots (No-op)
- * * Retrieve (return empty)
- * * Add (No-op)
- */
-private final boolean SKIP_TRACKER =
Boolean.getBoolean("oak.datastore.skipTracker");
-
private static final String datastoreMeta = "blobids";
private static final String fileNamePrefix = "blob";
private static final String mergedFileSuffix = ".refs";
private static final String snapshotMarkerSuffix = ".snapshot";
/* Local instance identifier */
-private final String instanceId = randomUUID().toString();
+private String instanceId = randomUUID().toString();
-private final SharedDataStore datastore;
-private final long snapshotInterval;
-private final ActiveDeletionTracker deleteTracker;
+private SharedDataStore datastore;
+private long snapshotInterval;
+private ActiveDeletionTracker deleteTracker;
protected BlobIdStore store;
-private final ScheduledExecutorService scheduler;
+private ScheduledExecutorService scheduler;
private String prefix;
private File rootDir;
-public BlobIdTracker(String path, String repositoryId, long
snapshotIntervalSecs, SharedDataStore datastore)
+private BlobIdTracker() {}
+
+private BlobIdTracker(String path, String repositoryId, long
snapshotIntervalSecs, SharedDataStore datastore)
throws IOException {
this(path, repositoryId, newSingleThreadScheduledExecutor(),
snapshotIntervalSecs, snapshotIntervalSecs,
datastore);
}
-public BlobIdTracker(String path, String repositoryId,
ScheduledExecutorService scheduler, long snapshotDelaySecs,
+private BlobIdTracker(String path, String repositoryId,
ScheduledExecutorService scheduler, long snapshotDelaySecs,
long snapshotIntervalSecs, SharedDataStore datastore) throw
svn commit: r1855477 - /jackrabbit/oak/trunk/oak-run/src/main/resources/logback-datastore.xml
Author: amitj
Date: Thu Mar 14 03:16:33 2019
New Revision: 1855477
URL: http://svn.apache.org/viewvc?rev=1855477&view=rev
Log:
OAK-8112: Change log pattern to have complete date for oak-run datastore command
Changed pattern to have the complete date
Modified:
jackrabbit/oak/trunk/oak-run/src/main/resources/logback-datastore.xml
Modified: jackrabbit/oak/trunk/oak-run/src/main/resources/logback-datastore.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/resources/logback-datastore.xml?rev=1855477&r1=1855476&r2=1855477&view=diff
==
--- jackrabbit/oak/trunk/oak-run/src/main/resources/logback-datastore.xml
(original)
+++ jackrabbit/oak/trunk/oak-run/src/main/resources/logback-datastore.xml Thu
Mar 14 03:16:33 2019
@@ -22,14 +22,14 @@
${oak.workDir}/datastore.log
- %d %-5level [%thread] %logger{30} %marker- %msg %n
+ %d{dd.MM. HH:mm:ss.SSS} %-5(*%level*) [%thread] %logger{30}
%marker- %msg %n
System.out
- %d{HH:mm:ss} - %msg%n
+ %d{dd.MM. HH:mm:ss.SSS} %-5(*%level*) [%thread] %logger{30}
%marker- %msg %n
svn commit: r1848912 - in /jackrabbit/oak/branches/1.8: ./ oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/
Author: amitj Date: Fri Dec 14 06:01:43 2018 New Revision: 1848912 URL: http://svn.apache.org/viewvc?rev=1848912&view=rev Log: OAK-7951: Datastore GC stats not updated with failure when "Not all repositories have marked references available" Merge r1848822,r1848823 from trunk Added: jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java - copied, changed from r1848822, jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java Modified: jackrabbit/oak/branches/1.8/ (props changed) jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java jackrabbit/oak/branches/1.8/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Propchange: jackrabbit/oak/branches/1.8/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Dec 14 06:01:43 2018 @@ -1,3 +1,3 @@ /jackrabbit/oak/branches/1.0:1665962 -/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911 ,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217 +/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911 ,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217,1848822-1848823 /ja
svn commit: r1848909 - in /jackrabbit/oak/branches/1.6: ./ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/ oak-core/src/te
Author: amitj Date: Fri Dec 14 03:41:40 2018 New Revision: 1848909 URL: http://svn.apache.org/viewvc?rev=1848909&view=rev Log: OAK-7951: Datastore GC stats not updated with failure when "Not all repositories have marked references available" Merge r1848822,r1848823 from trunk Added: jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java (with props) Modified: jackrabbit/oak/branches/1.6/ (props changed) jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java jackrabbit/oak/branches/1.6/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Propchange: jackrabbit/oak/branches/1.6/ -- --- svn:mergeinfo (original) +++ svn:mergeinfo Fri Dec 14 03:41:40 2018 @@ -1,4 +1,4 @@ /jackrabbit/oak/branches/1.0:1665962 /jackrabbit/oak/branches/1.8:1844835 -/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119 ,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808053,1808125,1808128,1808142,1808240,1808246,1808731,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814052,1814189,1814332,1814397,1814475,1815201,1815438,1815926,1816019,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1826957,1827472,1827486,1827977,1828502,1829527,1829569,1829587,1829665,1829854,1829864,1829987,1829998,1830019,1830160,1830239,1830748,1831190,1831374,1832258,1832379,1832535,1833308,1834112,1834648-1834649,1834681,1835060,1835780,1836082,1837475,1837998,1838 637,1839746,1840024,1840226,1842677,1843175,1843222,1843231,1844549,1844642,1844728,1845135,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848181-1848182,1848191,1848217 +/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785161,1785172,1785283,1785652,1785838,1785916-1785917,1785919,1785946,1786122,1787074,1787145,1787151,1787217,1787425,1788056,1788378,1788387-1788389,1788463,1788476,1788850,1789056,1789534,1789925,1789940,1789987,1790006,1790013,1790069,1790077,1790079,1790382,1790502-1790503,1792049,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1794417,1794683,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1797378,1798035,1798832,1798834,1799219,1799389,1799393,1799924,1800244,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119 ,1801675,1802260,1802262,1802286,1802548,1802905,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1804437,1805851-1805852,1806668,1807308,1807688,1808022,1808053,1808125,1808128,1808142,1808240,1808246,1808731,1809024,1809026,1809131,1809163,1809178-1809179,1809253,1809255-1809256,1809289,1809745,1811071-1811072,1811155,1811380,1811655,1811952,1811963,1811986,1813192,1813538,1814052,1814189,1814332,1814397,1814475,1815201,1815438,1815926,1816019,1817326,1817919,1817987-1817988,1817990,1818038,1818042,1818056,1818124,1818137,1818554,1818576,1818645,1819048,1819050,1821237,1821325,1821358,1821495,1821516,1821847,1822207,1822850,1823172,1823655,1824896,1825471,1825654,1826237,1826338,1826532,1826640,1826932,1
svn commit: r1848823 - /jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
Author: amitj Date: Thu Dec 13 06:37:00 2018 New Revision: 1848823 URL: http://svn.apache.org/viewvc?rev=1848823&view=rev Log: OAK-7951: Datastore GC stats not updated with failure when "Not all repositories have marked references available" - Missing License header Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java?rev=1848823&r1=1848822&r2=1848823&view=diff == --- jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java (original) +++ jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java Thu Dec 13 06:37:00 2018 @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.jackrabbit.oak.plugins.blob; import com.google.common.collect.ImmutableList;
svn commit: r1848822 - in /jackrabbit/oak/trunk/oak-blob-plugins/src: main/java/org/apache/jackrabbit/oak/plugins/blob/ test/java/org/apache/jackrabbit/oak/plugins/blob/
Author: amitj
Date: Thu Dec 13 06:34:07 2018
New Revision: 1848822
URL: http://svn.apache.org/viewvc?rev=1848822&view=rev
Log:
OAK-7951: Datastore GC stats not updated with failure when "Not all
repositories have marked references available"
Based on patch from Wim Symons
- Bubble up exception in case of not able to run sweep because of not all
repositories not having references available
- Mark failure in stats which can then be queried
Added:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1848822&r1=1848821&r2=1848822&view=diff
==
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Thu Dec 13 06:34:07 2018
@@ -85,10 +85,10 @@ import org.slf4j.LoggerFactory;
/**
* Mark and sweep garbage collector.
- *
+ *
* Uses the file system to store internal state while in process to account
for huge data.
* This class is not thread safe.
- *
+ *
*/
public class MarkSweepGarbageCollector implements BlobGarbageCollector {
@@ -97,7 +97,7 @@ public class MarkSweepGarbageCollector i
public static final String TEMP_DIR =
StandardSystemProperty.JAVA_IO_TMPDIR.value();
public static final int DEFAULT_BATCH_COUNT = 1024;
-
+
public static final String DELIM = ",";
private static final Function transformer = new
Function() {
@@ -234,7 +234,7 @@ public class MarkSweepGarbageCollector i
/**
* Returns the stats related to GC for all repos
- *
+ *
* @return a list of GarbageCollectionRepoStats objects
* @throws Exception
*/
@@ -348,16 +348,18 @@ public class MarkSweepGarbageCollector i
long deleteCount;
try {
deleteCount = sweep(fs, markStart, forceBlobRetrieve);
+
+long maxTime = getMaxModifiedTime(markStart) > 0 ?
getMaxModifiedTime(markStart) : markStart;
+LOG.info("Blob garbage collection completed in {} ({} ms).
Number of blobs deleted [{}] with max modification time of [{}]",
+sw.toString(), sw.elapsed(TimeUnit.MILLISECONDS),
deleteCount, timestampToString(maxTime));
+
threw = false;
+} catch (NotAllRepositoryMarkedException rm) {
+statsCollector.finishFailure();
} finally {
sw.stop();
statsCollector.updateSweepDuration(sw.elapsed(TimeUnit.MILLISECONDS) -
markFinish, TimeUnit.MILLISECONDS);
}
-
-long maxTime = getMaxModifiedTime(markStart) > 0 ?
getMaxModifiedTime(markStart) : markStart;
-
-LOG.info("Blob garbage collection completed in {} ({} ms).
Number of blobs deleted [{}] with max modification time of [{}]",
-sw.toString(), sw.elapsed(TimeUnit.MILLISECONDS),
deleteCount, timestampToString(maxTime));
}
} catch (Exception e) {
statsCollector.finishFailure();
@@ -394,7 +396,7 @@ public class MarkSweepGarbageCollector i
/**
* Difference phase where the GC candidates are identified.
- *
+ *
* @param fs the garbage collector file state
* @throws IOException
* Signals that an I/O exception has occurred.
@@ -453,14 +455,10 @@ public class MarkSweepGarbageCollector i
long earliestRefAvailTime;
// Merge all the blob references available from all the reference
files in the data store meta store
// Only go ahead if merge succeeded
-try {
-earliestRefAvailTime =
-
GarbageCollectionType.get(blobStore).mergeAllMarkedReferences(blobStore, fs);
-LOG.debug("Earliest reference available for timestamp [{}]",
earliestRefAvailTime);
-earliestRefAvailTime = (earliestRefAvailTime < markStart ?
earliestRefAvailTime : markStart);
-} catch (Exception e) {
-return 0;
-}
+earliestRefAvailTime =
+
GarbageCollectionType
