Author: amitj
Date: Thu Jun 23 06:00:30 2016
New Revision: 1749827
URL: http://svn.apache.org/viewvc?rev=1749827&view=rev
Log:
OAK-4429/OAK-4430
Merge r1747341, r1747342 from trunk
Modified:
jackrabbit/oak/branches/1.2/ (props changed)
jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
Propchange: jackrabbit/oak/branches/1.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 23 06:00:30 2016
@@ -1,4 +1,4 @@
/jackrabbit/oak/branches/1.0:1665962
/jackrabbit/oak/branches/1.4:1745750,1747354
-/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672611,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678202,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681921,1681955,1682042,1682218,1682235,1682437,1682488,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561
,1684570,1684601,1684618,1684669,1684820,1684868,1684894,1685023,1685075,1685370,1685541,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686003,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686772,1686780,1686790,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690657,1690669,1690672,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691201,1691210,1691217-1691218,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691394,1691401,1691498,1691509,1692133-1692134,1692156,1692250,1692272,1692274,1692363,1692382,1692393,1692478,1692955,1693002,1693030,1693050,1693209,1693401,1693421,1693525-1693526,1694
007,1694393-1694394,1694651,1694653-1694654,1695032,1695050,1695122,1695280,1695299,1695420,1695457,1695482,1695492,1695507,1695521,1695540,1695571,1695829-1695830,1695905,1696190,1696194,1696242,1696285,1696375,1696522,1696578,1696759,1696916,1697363,1697373,1697383,1697410,1697582,1697589,1697616,1697672,1697896,1700191,1700231,1700397,1700403,1700506,1700571,1700718,1700720,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701907,1701948,1701955,1701959,1701965,1701986,1702014,1702022,1702045,1702051,1702241,1702272,1702371,1702387,1702405,1702423,1702426,1702428,1702860,1702866,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703758,1703858,1703878,1704256,1704282,1704285,1704457,1704479,1704490,1704614,1704629,1704636,1704655,1704670,1704886,1705005,1705027,1705043,1705055,1705250,1705268,1705273,1705323,1705677,1705701,1705871,1705992,1705998,1706009,1706037,1706059,1706212,1706218,1706270,1
706764,1706772,1707049,1707189,1707191,1707331,1707435,1707509,1707753,1708049,1708105,1708307,1708315,1708546,1708592,1708766,1709012,1709852,1709978,1710013,1710031,1710049,1710205,1710242,1710559,1710575,1710590,1710614,1710637,1710789,1710800,1710811,1710816,1710972,1711248,1711282,1711296,1711405,1711498,1711654,1712018,1712042,1712319,1712490,1712531,1712730,1712785,1712963,1713008,1713439,1713461,1713580,1713586,1713599-1713600,1713626,1713698,1713803,1713809,1714034,1714061,1714084,1714170,1714213,1714229,1714238,1714519-1714520,1714543-1714544,1714730,1714739,1714779,1714956,1714961,1715010,1715092,1715191,1715346,1715716,1715767,1715771,1715888,1715898,1716100,1716178,1716426,1716576,1716588-1716589,1716596,1716616,1716703,1716712,1716815,1716823,1716830,1716883,1717203,1717277,1717410,1717462,1717632,1717768-1717769,1717784,1717789,1717988,1718528,1718533,1718547-1718548,1718626,1718646,1718772,1718801-1718802,1718895,1719111,1719288,1719869,1720306,1720335,1720350,172035
4,1720500,1721160,1721172,1721337,1722141,1722832,1723227,1723239,1723241,1723251,1723254,1723333,1723347,1723350,1723565,1723584,1723713,1723731,1724026,1724057,1724186,1724210,1724401,1724628,1724631,1725216,1725477,1725515,1725555,1725941,1725960,1726232,1726237,1726570,1726579,1726585-1726586,1726621,1726795,1726797,1726809,1726812,1726981,1726993,1727026,1727254,1727331,1727350,1727358,1727429,1727476,1727483,1727508,1727515-1727518,1727813,1727816,1727831-1727832,1727841,1727893,1727895,1727912-1727913,1727923,1727991,1728037,1728041,1728070,1728114,1728281,1728443,1728642,1729200,1729505,1729599,1729957,1729979,1730216,1730527,1730581,1730629,1730801,1731627,1731647-1731648,1731789,1731797,1732131,1732268,1732278,1732330,1732647-1732648,1732864,1733615,1733929,1734230,1734254,1735052,1735405,1735484,1735588,1736176,1737309-1737310,1737334,1737349,1738833,1738950,1738957,1739894,1740116,1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,174
4265,1744959,1745038,1745197,1746117,1746696,1746981,1747492,1748505,1748553,1748870,1749350,1749464,1749475,1749662
+/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672611,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678202,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681921,1681955,1682042,1682218,1682235,1682437,1682488,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561
,1684570,1684601,1684618,1684669,1684820,1684868,1684894,1685023,1685075,1685370,1685541,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686003,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686772,1686780,1686790,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690657,1690669,1690672,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691201,1691210,1691217-1691218,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691394,1691401,1691498,1691509,1692133-1692134,1692156,1692250,1692272,1692274,1692363,1692382,1692393,1692478,1692955,1693002,1693030,1693050,1693209,1693401,1693421,1693525-1693526,1694
007,1694393-1694394,1694651,1694653-1694654,1695032,1695050,1695122,1695280,1695299,1695420,1695457,1695482,1695492,1695507,1695521,1695540,1695571,1695829-1695830,1695905,1696190,1696194,1696242,1696285,1696375,1696522,1696578,1696759,1696916,1697363,1697373,1697383,1697410,1697582,1697589,1697616,1697672,1697896,1700191,1700231,1700397,1700403,1700506,1700571,1700718,1700720,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701907,1701948,1701955,1701959,1701965,1701986,1702014,1702022,1702045,1702051,1702241,1702272,1702371,1702387,1702405,1702423,1702426,1702428,1702860,1702866,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703758,1703858,1703878,1704256,1704282,1704285,1704457,1704479,1704490,1704614,1704629,1704636,1704655,1704670,1704886,1705005,1705027,1705043,1705055,1705250,1705268,1705273,1705323,1705677,1705701,1705871,1705992,1705998,1706009,1706037,1706059,1706212,1706218,1706270,1
706764,1706772,1707049,1707189,1707191,1707331,1707435,1707509,1707753,1708049,1708105,1708307,1708315,1708546,1708592,1708766,1709012,1709852,1709978,1710013,1710031,1710049,1710205,1710242,1710559,1710575,1710590,1710614,1710637,1710789,1710800,1710811,1710816,1710972,1711248,1711282,1711296,1711405,1711498,1711654,1712018,1712042,1712319,1712490,1712531,1712730,1712785,1712963,1713008,1713439,1713461,1713580,1713586,1713599-1713600,1713626,1713698,1713803,1713809,1714034,1714061,1714084,1714170,1714213,1714229,1714238,1714519-1714520,1714543-1714544,1714730,1714739,1714779,1714956,1714961,1715010,1715092,1715191,1715346,1715716,1715767,1715771,1715888,1715898,1716100,1716178,1716426,1716576,1716588-1716589,1716596,1716616,1716703,1716712,1716815,1716823,1716830,1716883,1717203,1717277,1717410,1717462,1717632,1717768-1717769,1717784,1717789,1717988,1718528,1718533,1718547-1718548,1718626,1718646,1718772,1718801-1718802,1718895,1719111,1719288,1719869,1720306,1720335,1720350,172035
4,1720500,1721160,1721172,1721337,1722141,1722832,1723227,1723239,1723241,1723251,1723254,1723333,1723347,1723350,1723565,1723584,1723713,1723731,1724026,1724057,1724186,1724210,1724401,1724628,1724631,1725216,1725477,1725515,1725555,1725941,1725960,1726232,1726237,1726570,1726579,1726585-1726586,1726621,1726795,1726797,1726809,1726812,1726981,1726993,1727026,1727254,1727331,1727350,1727358,1727429,1727476,1727483,1727508,1727515-1727518,1727813,1727816,1727831-1727832,1727841,1727893,1727895,1727912-1727913,1727923,1727991,1728037,1728041,1728070,1728114,1728281,1728443,1728642,1729200,1729505,1729599,1729957,1729979,1730216,1730527,1730581,1730629,1730801,1731627,1731647-1731648,1731789,1731797,1732131,1732268,1732278,1732330,1732647-1732648,1732864,1733615,1733929,1734230,1734254,1735052,1735405,1735484,1735588,1736176,1737309-1737310,1737334,1737349,1738833,1738950,1738957,1739894,1740116,1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,174
4265,1744959,1745038,1745197,1746117,1746696,1746981,1747341-1747342,1747492,1748505,1748553,1748870,1749350,1749464,1749475,1749662
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml (original)
+++ jackrabbit/oak/branches/1.2/oak-blob-cloud/pom.xml Thu Jun 23 06:00:30 2016
@@ -124,6 +124,10 @@
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
Modified:
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/S3Backend.java
Thu Jun 23 06:00:30 2016
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
+import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -34,6 +35,10 @@ import java.util.concurrent.ThreadPoolEx
import java.util.concurrent.TimeUnit;
import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.core.data.AsyncTouchCallback;
import org.apache.jackrabbit.core.data.AsyncTouchResult;
import org.apache.jackrabbit.core.data.AsyncUploadCallback;
@@ -66,6 +71,9 @@ import com.amazonaws.services.s3.transfe
import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.util.StringUtils;
+import static com.google.common.collect.Iterables.filter;
+import static java.lang.Thread.currentThread;
+
/**
* A data store backend that stores data on Amazon S3.
*/
@@ -409,33 +417,13 @@ public class S3Backend implements Shared
@Override
public Iterator<DataIdentifier> getAllIdentifiers()
throws DataStoreException {
- long start = System.currentTimeMillis();
- ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(
- getClass().getClassLoader());
- Set<DataIdentifier> ids = new HashSet<DataIdentifier>();
- ObjectListing prevObjectListing = s3service.listObjects(bucket);
- while (true) {
- for (S3ObjectSummary s3ObjSumm :
prevObjectListing.getObjectSummaries()) {
- String id = getIdentifierName(s3ObjSumm.getKey());
- if (id != null && !id.startsWith(META_KEY_PREFIX)) {
- ids.add(new DataIdentifier(id));
- }
+ return new RecordsIterator<DataIdentifier>(
+ new Function<S3ObjectSummary, DataIdentifier>() {
+ @Override
+ public DataIdentifier apply(S3ObjectSummary input) {
+ return new
DataIdentifier(getIdentifierName(input.getKey()));
}
- if (!prevObjectListing.isTruncated()) break;
- prevObjectListing =
s3service.listNextBatchOfObjects(prevObjectListing);
- }
- LOG.debug("getAllIdentifiers returned size [{}] took [{}] ms.",
- ids.size(), (System.currentTimeMillis() - start));
- return ids.iterator();
- } catch (AmazonServiceException e) {
- throw new DataStoreException("Could not list objects", e);
- } finally {
- if (contextClassLoader != null) {
-
Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
- }
+ });
}
@Override
@@ -703,6 +691,93 @@ public class S3Backend implements Shared
}
}
+ @Override
+ public Iterator<DataRecord> getAllRecords() {
+ return new RecordsIterator<DataRecord>(
+ new Function<S3ObjectSummary, DataRecord>() {
+ @Override
+ public DataRecord apply(S3ObjectSummary input) {
+ return new S3DataRecord(s3service, bucket,
getIdentifierName(input.getKey()),
+ input.getLastModified().getTime(), input.getSize());
+ }
+ });
+ }
+
+ /**
+ * Returns an iterator over the S3 objects
+ * @param <T>
+ */
+ class RecordsIterator<T> extends AbstractIterator<T> {
+ ObjectListing prevObjectListing;
+ Queue<S3ObjectSummary> queue;
+ long size;
+ Function<S3ObjectSummary, T> transformer;
+
+ public RecordsIterator (Function<S3ObjectSummary, T> transformer) {
+ queue = Lists.newLinkedList();
+ this.transformer = transformer;
+ }
+
+ @Override
+ protected T computeNext() {
+ if (queue.isEmpty()) {
+ loadBatch();
+ }
+
+ if (!queue.isEmpty()) {
+ return transformer.apply(queue.remove());
+ }
+
+ return endOfData();
+ }
+
+ private boolean loadBatch() {
+ ClassLoader contextClassLoader =
currentThread().getContextClassLoader();
+ long start = System.currentTimeMillis();
+ try {
+
currentThread().setContextClassLoader(getClass().getClassLoader());
+
+ // initialize the listing the first time
+ if (prevObjectListing == null) {
+ prevObjectListing = s3service.listObjects(bucket);
+ } else if (prevObjectListing.isTruncated()) { //already
initialized more objects available
+ prevObjectListing =
s3service.listNextBatchOfObjects(prevObjectListing);
+ } else { // no more available
+ return false;
+ }
+
+ List<S3ObjectSummary> listing = Lists.newArrayList(
+ filter(prevObjectListing.getObjectSummaries(),
+ new Predicate<S3ObjectSummary>() {
+ @Override
+ public boolean apply(S3ObjectSummary input) {
+ return
!input.getKey().startsWith(META_KEY_PREFIX);
+ }
+ }));
+
+ // After filtering no elements
+ if (listing.isEmpty()) {
+ return false;
+ }
+
+ size += listing.size();
+ queue.addAll(listing);
+
+ LOG.info("Loaded batch of size [{}] in [{}] ms.",
+ listing.size(), (System.currentTimeMillis() - start));
+
+ return true;
+ } catch (AmazonServiceException e) {
+ LOG.warn("Could not list objects", e);
+ } finally {
+ if (contextClassLoader != null) {
+ currentThread().setContextClassLoader(contextClassLoader);
+ }
+ }
+ return false;
+ }
+ }
+
private static String addMetaKeyPrefix(String key) {
return META_KEY_PREFIX + key;
}
Modified:
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/aws/s3/SharedS3Backend.java
Thu Jun 23 06:00:30 2016
@@ -24,6 +24,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataStoreException;
import java.io.InputStream;
+import java.util.Iterator;
import java.util.List;
/**
@@ -70,4 +71,13 @@ public interface SharedS3Backend extends
* @param prefix the prefix of the record
*/
void deleteAllMetadataRecords(String prefix);
+
+ /**
+ * Returns a list of all DataRecords
+ *
+ * @return iterator over DataRecords
+ * @throws DataStoreException
+ */
+ Iterator<DataRecord> getAllRecords()
+ throws DataStoreException;
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStore.java
Thu Jun 23 06:00:30 2016
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.plugins.blob;
import java.io.InputStream;
+import java.util.Iterator;
import java.util.List;
import org.apache.jackrabbit.core.data.DataRecord;
@@ -75,6 +76,13 @@ public interface SharedDataStore {
void deleteAllMetadataRecords(String prefix);
/**
+ * Retrieved an iterator over all DataRecords.
+ *
+ * @return iterator over DataRecords
+ */
+ Iterator<DataRecord> getAllRecords();
+
+ /**
* Gets the type.
*
* @return the type
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Thu Jun 23 06:00:30 2016
@@ -346,18 +346,7 @@ public class DataStoreBlobStore implemen
@Override
public Iterator<String> getAllChunkIds(final long maxLastModifiedTime)
throws Exception {
- return transform(filter(transform(delegate.getAllIdentifiers(), new
Function<DataIdentifier, DataRecord>() {
- @Nullable
- @Override
- public DataRecord apply(@Nullable DataIdentifier input) {
- try {
- return delegate.getRecord(input);
- } catch (DataStoreException e) {
- log.warn("Error occurred while fetching DataRecord for
identifier {}", input, e);
- }
- return null;
- }
- }), new Predicate<DataRecord>() {
+ return transform(filter(getAllRecords(), new Predicate<DataRecord>() {
@Override
public boolean apply(@Nullable DataRecord input) {
if (input != null && (maxLastModifiedTime <= 0
@@ -447,6 +436,14 @@ public class DataStoreBlobStore implemen
}
@Override
+ public Iterator<DataRecord> getAllRecords() {
+ if (delegate instanceof SharedDataStore) {
+ return ((SharedDataStore) delegate).getAllRecords();
+ }
+ return Iterators.emptyIterator();
+ }
+
+ @Override
public Type getType() {
if (delegate instanceof SharedDataStore) {
return Type.SHARED;
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStore.java
Thu Jun 23 06:00:30 2016
@@ -218,6 +218,26 @@ public class OakFileDataStore extends Fi
}
@Override
+ public Iterator<DataRecord> getAllRecords() {
+ final String path = normalizeNoEndSeparator(getPath());
+ final OakFileDataStore store = this;
+ return Files.fileTreeTraverser().postOrderTraversal(new
File(getPath()))
+ .filter(new Predicate<File>() {
+ @Override
+ public boolean apply(File input) {
+ return input.isFile() &&
+
!normalizeNoEndSeparator(input.getParent()).equals(path);
+ }
+ })
+ .transform(new Function<File, DataRecord>() {
+ @Override
+ public DataRecord apply(File input) {
+ return new FileDataRecord(store, new
DataIdentifier(input.getName()), input);
+ }
+ }).iterator();
+ }
+
+ @Override
public Type getType() {
return Type.SHARED;
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedS3DataStore.java
Thu Jun 23 06:00:30 2016
@@ -26,6 +26,7 @@ import org.apache.jackrabbit.oak.blob.cl
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import java.io.InputStream;
+import java.util.Iterator;
import java.util.List;
/**
@@ -69,6 +70,11 @@ public class SharedS3DataStore extends S
}
@Override
+ public Iterator<DataRecord> getAllRecords() {
+ return backend.getAllRecords();
+ }
+
+ @Override
public Type getType() {
return Type.SHARED;
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java
Thu Jun 23 06:00:30 2016
@@ -18,37 +18,53 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import static com.google.common.collect.Sets.newHashSet;
+import static junit.framework.Assert.assertEquals;
+import static
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.cleanup;
+import static
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.getBlobStore;
import static org.hamcrest.CoreMatchers.instanceOf;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+import java.util.Random;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.junit.After;
import org.junit.Assume;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Test for SharedDataUtils to test addition, retrieval and deletion of root
records.
*/
public class SharedDataStoreUtilsTest {
- SharedDataStore dataStore;
+ private static final Logger log =
LoggerFactory.getLogger(SharedDataStoreUtilsTest.class);
+
+ DataStoreBlobStore dataStore;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
try {
- Assume.assumeThat(DataStoreUtils.getBlobStore(),
instanceOf(SharedDataStore.class));
+ Assume.assumeThat(getBlobStore(),
instanceOf(SharedDataStore.class));
} catch (Exception e) {
Assume.assumeNoException(e);
}
@@ -56,7 +72,7 @@ public class SharedDataStoreUtilsTest {
@Test
public void test() throws Exception {
- dataStore = DataStoreUtils.getBlobStore();
+ dataStore = getBlobStore();
String repoId1 = UUID.randomUUID().toString();
String repoId2 = UUID.randomUUID().toString();
@@ -72,7 +88,7 @@ public class SharedDataStoreUtilsTest {
dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]),
SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1));
DataRecord markerRec1 =
dataStore.getMetadataRecord(SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1));
- Assert.assertEquals(
+ assertEquals(
SharedStoreRecordType.MARKED_START_MARKER.getIdFromName(markerRec1.getIdentifier().toString()),
repoId1);
long lastModifiedMarkerRec1 = markerRec1.getLastModified();
@@ -89,14 +105,14 @@ public class SharedDataStoreUtilsTest {
DataRecord rec2 =
dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId2));
long lastModifiedRec2 = rec2.getLastModified();
-
Assert.assertEquals(SharedStoreRecordType.REPOSITORY.getIdFromName(repo1.getIdentifier().toString()),
repoId1);
- Assert.assertEquals(
+
assertEquals(SharedStoreRecordType.REPOSITORY.getIdFromName(repo1.getIdentifier().toString()),
repoId1);
+ assertEquals(
SharedStoreRecordType.REPOSITORY.getIdFromName(repo2.getIdentifier().toString()),
repoId2);
- Assert.assertEquals(
+ assertEquals(
SharedStoreRecordType.REFERENCES.getIdFromName(rec1.getIdentifier().toString()),
repoId1);
- Assert.assertEquals(
+ assertEquals(
SharedStoreRecordType.REFERENCES.getIdFromName(rec2.getIdentifier().toString()),
repoId2);
@@ -108,7 +124,7 @@ public class SharedDataStoreUtilsTest {
// Since, we don't care about which file specifically but only the
earliest timestamped record
// Earliest time should be the min timestamp from the 2 reference files
long minRefTime = (lastModifiedRec1 <= lastModifiedRec2 ?
lastModifiedRec1 : lastModifiedRec2);
- Assert.assertEquals(
+ assertEquals(
SharedDataStoreUtils.getEarliestRecord(
dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())).getLastModified(),
minRefTime);
@@ -129,13 +145,59 @@ public class SharedDataStoreUtilsTest {
Assert.assertTrue(dataStore.getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType()).isEmpty());
// Repository ids should still be available
- Assert.assertEquals(2,
+ assertEquals(2,
dataStore.getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()).size());
}
+ @Test
+ public void testGetAllChunkIds() throws Exception {
+ dataStore = getBlobStore();
+ int number = 1010;
+ Set<String> added = newHashSet();
+ for (int i = 0; i < number; i++) {
+ String rec = dataStore.writeBlob(randomStream(i, 16516));
+ added.add(rec);
+ }
+
+ Set<String> retrieved = newHashSet(dataStore.getAllChunkIds(0));
+ assertEquals(added, retrieved);
+ }
+
+ @Test
+ public void testGetAllRecords() throws Exception {
+ dataStore = getBlobStore();
+ int number = 1010;
+ Set<String> added = newHashSet();
+ for (int i = 0; i < number; i++) {
+ String rec = dataStore.addRecord(randomStream(i, 16516))
+ .getIdentifier().toString();
+ added.add(rec);
+ }
+
+ Set<String> retrieved =
newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()),
+ new Function<DataRecord, String>() {
+ @Nullable @Override public String apply(@Nullable DataRecord
input) {
+ return input.getIdentifier().toString();
+ }
+ }));
+ assertEquals(added, retrieved);
+ }
+
+ static InputStream randomStream(int seed, int size) {
+ Random r = new Random(seed);
+ byte[] data = new byte[size];
+ r.nextBytes(data);
+ return new ByteArrayInputStream(data);
+ }
+
@After
public void close() throws IOException {
FileUtils.cleanDirectory(new File(DataStoreUtils.getHomeDir()));
+ try {
+ cleanup(dataStore.getDataStore(), new Date());
+ } catch (Exception e) {
+ log.error("Error closing data store", e);
+ }
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java
Thu Jun 23 06:00:30 2016
@@ -29,7 +29,12 @@ import java.util.List;
import java.util.Random;
import java.util.Set;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -39,6 +44,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStoreTest;
import org.apache.jackrabbit.oak.spi.blob.BlobStoreInputStream;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -147,9 +153,16 @@ public class DataStoreBlobStoreTest exte
DataIdentifier d20 = new DataIdentifier("d-20");
DataIdentifier d30 = new DataIdentifier("d-30");
List<DataIdentifier> dis = ImmutableList.of(d10, d20, d30);
-
- DataStore mockedDS = mock(DataStore.class);
- when(mockedDS.getAllIdentifiers()).thenReturn(dis.iterator());
+ List<DataRecord> recs = Lists.newArrayList(
+ Iterables.transform(dis, new Function<DataIdentifier,
DataRecord>() {
+ @Nullable
+ @Override
+ public DataRecord apply(@Nullable DataIdentifier input) {
+ return new TimeDataRecord(input);
+ }
+ }));
+ OakFileDataStore mockedDS = mock(OakFileDataStore.class);
+ when(mockedDS.getAllRecords()).thenReturn(recs.iterator());
when(mockedDS.getRecord(new DataIdentifier("d-10"))).thenReturn(new
TimeDataRecord(d10));
when(mockedDS.getRecord(new DataIdentifier("d-20"))).thenReturn(new
TimeDataRecord(d20));
when(mockedDS.getRecord(new DataIdentifier("d-30"))).thenReturn(new
TimeDataRecord(d30));
@@ -158,7 +171,6 @@ public class DataStoreBlobStoreTest exte
Iterator<String> chunks = ds.getAllChunkIds(25);
Set<String> expected = Sets.newHashSet("d-10","d-20");
assertEquals(expected, Sets.newHashSet(chunks));
-
}
@Test
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java?rev=1749827&r1=1749826&r2=1749827&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreUtils.java
Thu Jun 23 06:00:30 2016
@@ -97,7 +97,7 @@ public class DataStoreUtils {
}
public static FileDataStore createFDS(File root, int minRecordLength) {
- FileDataStore fds = new FileDataStore();
+ OakFileDataStore fds = new OakFileDataStore();
fds.setPath(root.getAbsolutePath());
fds.setMinRecordLength(minRecordLength);
fds.init(null);
