This is an automated email from the ASF dual-hosted git repository.
gaul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/jclouds.git
The following commit(s) were added to refs/heads/master by this push:
new 29eec44 JCLOUDS-1371: JCLOUDS-1488: list optimize prefix
29eec44 is described below
commit 29eec441e902162394a5dbceaf98c14d3b2bc87a
Author: Andrew Gaul <[email protected]>
AuthorDate: Fri Jan 25 11:42:13 2019 -0800
JCLOUDS-1371: JCLOUDS-1488: list optimize prefix
Previously getBlobKeysInsideContainer returned all keys and filtered
in LocalBlobStore. Now getBlobKeysInsideContainer filters via prefix
which can dramatically decrease the number of keys returned,
especially for the filesystem provider. Further optimizations are
possible for delimiter.
---
.../internal/FilesystemStorageStrategyImpl.java | 24 +++++++++++++++-------
.../FilesystemStorageStrategyImplTest.java | 8 ++++----
.../jclouds/blobstore/LocalStorageStrategy.java | 2 +-
.../blobstore/TransientStorageStrategy.java | 14 +++++++++----
.../jclouds/blobstore/config/LocalBlobStore.java | 4 ++--
5 files changed, 34 insertions(+), 18 deletions(-)
diff --git
a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
index a9f0a9c..90876df 100644
---
a/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
+++
b/apis/filesystem/src/main/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImpl.java
@@ -342,7 +342,7 @@ public class FilesystemStorageStrategyImpl implements
LocalStorageStrategy {
* @throws IOException
*/
@Override
- public Iterable<String> getBlobKeysInsideContainer(String container) throws
IOException {
+ public Iterable<String> getBlobKeysInsideContainer(String container, String
prefix) throws IOException {
filesystemContainerNameValidator.validate(container);
// check if container exists
// TODO maybe an error is more appropriate
@@ -353,7 +353,7 @@ public class FilesystemStorageStrategyImpl implements
LocalStorageStrategy {
File containerFile = openFolder(container);
final int containerPathLength = containerFile.getAbsolutePath().length()
+ 1;
- populateBlobKeysInContainer(containerFile, blobNames, new
Function<String, String>() {
+ populateBlobKeysInContainer(containerFile, blobNames, prefix, new
Function<String, String>() {
@Override
public String apply(String string) {
return denormalize(string.substring(containerPathLength));
@@ -753,7 +753,7 @@ public class FilesystemStorageStrategyImpl implements
LocalStorageStrategy {
public long countBlobs(String container, ListContainerOptions options) {
// TODO: honor options
try {
- return Iterables.size(getBlobKeysInsideContainer(container));
+ return Iterables.size(getBlobKeysInsideContainer(container, null));
} catch (IOException ioe) {
throw Throwables.propagate(ioe);
}
@@ -964,17 +964,27 @@ public class FilesystemStorageStrategyImpl implements
LocalStorageStrategy {
}
private static void populateBlobKeysInContainer(File directory, Set<String>
blobNames,
- Function<String, String> function) {
+ String prefix, Function<String, String> function) {
File[] children = directory.listFiles();
if (children == null) {
return;
}
for (File child : children) {
+ String fullPath = function.apply(child.getAbsolutePath());
if (child.isFile()) {
- blobNames.add( function.apply(child.getAbsolutePath()) );
+ if (prefix != null && !fullPath.startsWith(prefix)) {
+ continue;
+ }
+ blobNames.add(fullPath);
} else if (child.isDirectory()) {
- blobNames.add(function.apply(child.getAbsolutePath()) +
File.separator); // TODO: undo if failures
- populateBlobKeysInContainer(child, blobNames, function);
+ // Consider a prefix /a/b/c but we have only descended to path /a.
+ // We need to match the path against the prefix to continue
+ // matching down to /a/b.
+ if (prefix != null && !fullPath.startsWith(prefix) &&
!prefix.startsWith(fullPath + "/")) {
+ continue;
+ }
+ blobNames.add(fullPath + File.separator); // TODO: undo if failures
+ populateBlobKeysInContainer(child, blobNames, prefix, function);
}
}
}
diff --git
a/apis/filesystem/src/test/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImplTest.java
b/apis/filesystem/src/test/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImplTest.java
index 7b4cb88..33157ce 100644
---
a/apis/filesystem/src/test/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImplTest.java
+++
b/apis/filesystem/src/test/java/org/jclouds/filesystem/strategy/internal/FilesystemStorageStrategyImplTest.java
@@ -427,7 +427,7 @@ public class FilesystemStorageStrategyImplTest {
Blob blob = storageStrategy.newBlob(blobKey);
storageStrategy.putBlob(CONTAINER_NAME, blob);
- Iterable<String> keys =
storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME);
+ Iterable<String> keys =
storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME, null);
Iterator<String> iter = keys.iterator();
assertTrue(iter.hasNext());
assertEquals(iter.next(), blobKey);
@@ -598,7 +598,7 @@ public class FilesystemStorageStrategyImplTest {
Iterable<String> resultList;
// no container
- resultList = storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME);
+ resultList = storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME,
null);
assertNotNull(resultList, "Result is null");
assertFalse(resultList.iterator().hasNext(), "Blobs detected");
@@ -609,10 +609,10 @@ public class FilesystemStorageStrategyImplTest {
TestUtils.createRandomBlobKey("GetBlobKeys-", ".jpg"),
TestUtils.createRandomBlobKey("563" + "/" + "g3sx2" + "/" +
"removeBlob-", ".jpg"),
TestUtils.createRandomBlobKey("563" + "/" + "g3sx2" + "/" +
"removeBlob-", ".jpg") });
- storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME);
+ storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME, null);
List<String> retrievedBlobKeys = Lists.newArrayList();
- resultList = storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME);
+ resultList = storageStrategy.getBlobKeysInsideContainer(CONTAINER_NAME,
null);
Iterator<String> containersIterator = resultList.iterator();
while (containersIterator.hasNext()) {
retrievedBlobKeys.add(containersIterator.next());
diff --git
a/blobstore/src/main/java/org/jclouds/blobstore/LocalStorageStrategy.java
b/blobstore/src/main/java/org/jclouds/blobstore/LocalStorageStrategy.java
index 1498ee1..1a0ebd8 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/LocalStorageStrategy.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/LocalStorageStrategy.java
@@ -98,7 +98,7 @@ public interface LocalStorageStrategy {
* @return
* @throws IOException
*/
- Iterable<String> getBlobKeysInsideContainer(String container) throws
IOException;
+ Iterable<String> getBlobKeysInsideContainer(String container, String
prefix) throws IOException;
/**
* Load the blob with the given key belonging to the container with the
given
diff --git
a/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
b/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
index bd662f3..3053aa4 100644
---
a/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
+++
b/blobstore/src/main/java/org/jclouds/blobstore/TransientStorageStrategy.java
@@ -25,6 +25,7 @@ import java.util.Date;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import javax.inject.Inject;
@@ -59,7 +60,7 @@ import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
public class TransientStorageStrategy implements LocalStorageStrategy {
- private final ConcurrentMap<String, ConcurrentMap<String, Blob>>
containerToBlobs = new ConcurrentHashMap<String, ConcurrentMap<String, Blob>>();
+ private final ConcurrentMap<String, ConcurrentSkipListMap<String, Blob>>
containerToBlobs = new ConcurrentHashMap<String, ConcurrentSkipListMap<String,
Blob>>();
private final ConcurrentMap<String, ConcurrentMap<String, BlobAccess>>
containerToBlobAccess = new ConcurrentHashMap<String, ConcurrentMap<String,
BlobAccess>>();
private final ConcurrentMap<String, StorageMetadata> containerMetadata =
new ConcurrentHashMap<String, StorageMetadata>();
private final ConcurrentMap<String, ContainerAccess> containerAccessMap =
new ConcurrentHashMap<String, ContainerAccess>();
@@ -90,7 +91,7 @@ public class TransientStorageStrategy implements
LocalStorageStrategy {
@Override
public boolean createContainerInLocation(String containerName, Location
location, CreateContainerOptions options) {
ConcurrentMap<String, Blob> origValue = containerToBlobs.putIfAbsent(
- containerName, new ConcurrentHashMap<String, Blob>());
+ containerName, new ConcurrentSkipListMap<String, Blob>());
if (origValue != null) {
return false;
}
@@ -148,8 +149,13 @@ public class TransientStorageStrategy implements
LocalStorageStrategy {
}
@Override
- public Iterable<String> getBlobKeysInsideContainer(final String
containerName) {
- return containerToBlobs.get(containerName).keySet();
+ public Iterable<String> getBlobKeysInsideContainer(final String
containerName, String prefix) {
+ ConcurrentSkipListMap<String, Blob> blobs =
containerToBlobs.get(containerName);
+ if (prefix == null) {
+ return blobs.keySet();
+ }
+ String lastPrefix = prefix + (char) 65535; // TODO: better sentinel?
+ return blobs.subMap(prefix, /*fromInclusive=*/ true, lastPrefix,
/*toInclusive=*/ false).keySet();
}
@Override
diff --git
a/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java
b/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java
index 3717ed4..b62ad18 100644
--- a/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java
+++ b/blobstore/src/main/java/org/jclouds/blobstore/config/LocalBlobStore.java
@@ -238,7 +238,7 @@ public final class LocalBlobStore implements BlobStore {
// Loading blobs from container
Iterable<String> blobBelongingToContainer = null;
try {
- blobBelongingToContainer =
storageStrategy.getBlobKeysInsideContainer(containerName);
+ blobBelongingToContainer =
storageStrategy.getBlobKeysInsideContainer(containerName, options.getPrefix());
} catch (IOException e) {
logger.error(e, "An error occurred loading blobs contained into
container %s", containerName);
propagate(e);
@@ -414,7 +414,7 @@ public final class LocalBlobStore implements BlobStore {
boolean returnVal = true;
if (storageStrategy.containerExists(containerName)) {
try {
- if
(Iterables.isEmpty(storageStrategy.getBlobKeysInsideContainer(containerName)))
+ if
(Iterables.isEmpty(storageStrategy.getBlobKeysInsideContainer(containerName,
null)))
storageStrategy.deleteContainer(containerName);
else
returnVal = false;
